diff --git a/Documentation/admin-guide/cgroup-v1/hugetlb.rst b/Documentation/admin-guide/cgroup-v1/hugetlb.rst index a3902aa253a9..338f2c7d7a1c 100644 --- a/Documentation/admin-guide/cgroup-v1/hugetlb.rst +++ b/Documentation/admin-guide/cgroup-v1/hugetlb.rst @@ -2,13 +2,6 @@ HugeTLB Controller ================== -The HugeTLB controller allows to limit the HugeTLB usage per control group and -enforces the controller limit during page fault. Since HugeTLB doesn't -support page reclaim, enforcing the limit at page fault time implies that, -the application will get SIGBUS signal if it tries to access HugeTLB pages -beyond its limit. This requires the application to know beforehand how much -HugeTLB pages it would require for its use. - HugeTLB controller can be created by first mounting the cgroup filesystem. # mount -t cgroup -o hugetlb none /sys/fs/cgroup @@ -28,10 +21,14 @@ process (bash) into it. Brief summary of control files:: - hugetlb..limit_in_bytes # set/show limit of "hugepagesize" hugetlb usage - hugetlb..max_usage_in_bytes # show max "hugepagesize" hugetlb usage recorded - hugetlb..usage_in_bytes # show current usage for "hugepagesize" hugetlb - hugetlb..failcnt # show the number of allocation failure due to HugeTLB limit + hugetlb..rsvd.limit_in_bytes # set/show limit of "hugepagesize" hugetlb reservations + hugetlb..rsvd.max_usage_in_bytes # show max "hugepagesize" hugetlb reservations and no-reserve faults + hugetlb..rsvd.usage_in_bytes # show current reservations and no-reserve faults for "hugepagesize" hugetlb + hugetlb..rsvd.failcnt # show the number of allocation failure due to HugeTLB reservation limit + hugetlb..limit_in_bytes # set/show limit of "hugepagesize" hugetlb faults + hugetlb..max_usage_in_bytes # show max "hugepagesize" hugetlb usage recorded + hugetlb..usage_in_bytes # show current usage for "hugepagesize" hugetlb + hugetlb..failcnt # show the number of allocation failure due to HugeTLB usage limit For a system supporting three hugepage sizes (64k, 32M and 1G), the control files include:: @@ -40,11 +37,95 @@ files include:: hugetlb.1GB.max_usage_in_bytes hugetlb.1GB.usage_in_bytes hugetlb.1GB.failcnt + hugetlb.1GB.rsvd.limit_in_bytes + hugetlb.1GB.rsvd.max_usage_in_bytes + hugetlb.1GB.rsvd.usage_in_bytes + hugetlb.1GB.rsvd.failcnt hugetlb.64KB.limit_in_bytes hugetlb.64KB.max_usage_in_bytes hugetlb.64KB.usage_in_bytes hugetlb.64KB.failcnt + hugetlb.64KB.rsvd.limit_in_bytes + hugetlb.64KB.rsvd.max_usage_in_bytes + hugetlb.64KB.rsvd.usage_in_bytes + hugetlb.64KB.rsvd.failcnt hugetlb.32MB.limit_in_bytes hugetlb.32MB.max_usage_in_bytes hugetlb.32MB.usage_in_bytes hugetlb.32MB.failcnt + hugetlb.32MB.rsvd.limit_in_bytes + hugetlb.32MB.rsvd.max_usage_in_bytes + hugetlb.32MB.rsvd.usage_in_bytes + hugetlb.32MB.rsvd.failcnt + + +1. Page fault accounting + +hugetlb..limit_in_bytes +hugetlb..max_usage_in_bytes +hugetlb..usage_in_bytes +hugetlb..failcnt + +The HugeTLB controller allows users to limit the HugeTLB usage (page fault) per +control group and enforces the limit during page fault. Since HugeTLB +doesn't support page reclaim, enforcing the limit at page fault time implies +that, the application will get SIGBUS signal if it tries to fault in HugeTLB +pages beyond its limit. Therefore the application needs to know exactly how many +HugeTLB pages it uses before hand, and the sysadmin needs to make sure that +there are enough available on the machine for all the users to avoid processes +getting SIGBUS. + + +2. Reservation accounting + +hugetlb..rsvd.limit_in_bytes +hugetlb..rsvd.max_usage_in_bytes +hugetlb..rsvd.usage_in_bytes +hugetlb..rsvd.failcnt + +The HugeTLB controller allows to limit the HugeTLB reservations per control +group and enforces the controller limit at reservation time and at the fault of +HugeTLB memory for which no reservation exists. Since reservation limits are +enforced at reservation time (on mmap or shget), reservation limits never causes +the application to get SIGBUS signal if the memory was reserved before hand. For +MAP_NORESERVE allocations, the reservation limit behaves the same as the fault +limit, enforcing memory usage at fault time and causing the application to +receive a SIGBUS if it's crossing its limit. + +Reservation limits are superior to page fault limits described above, since +reservation limits are enforced at reservation time (on mmap or shget), and +never causes the application to get SIGBUS signal if the memory was reserved +before hand. This allows for easier fallback to alternatives such as +non-HugeTLB memory for example. In the case of page fault accounting, it's very +hard to avoid processes getting SIGBUS since the sysadmin needs precisely know +the HugeTLB usage of all the tasks in the system and make sure there is enough +pages to satisfy all requests. Avoiding tasks getting SIGBUS on overcommited +systems is practically impossible with page fault accounting. + + +3. Caveats with shared memory + +For shared HugeTLB memory, both HugeTLB reservation and page faults are charged +to the first task that causes the memory to be reserved or faulted, and all +subsequent uses of this reserved or faulted memory is done without charging. + +Shared HugeTLB memory is only uncharged when it is unreserved or deallocated. +This is usually when the HugeTLB file is deleted, and not when the task that +caused the reservation or fault has exited. + + +4. Caveats with HugeTLB cgroup offline. + +When a HugeTLB cgroup goes offline with some reservations or faults still +charged to it, the behavior is as follows: + +- The fault charges are charged to the parent HugeTLB cgroup (reparented), +- the reservation charges remain on the offline HugeTLB cgroup. + +This means that if a HugeTLB cgroup gets offlined while there is still HugeTLB +reservations charged to it, that cgroup persists as a zombie until all HugeTLB +reservations are uncharged. HugeTLB reservations behave in this manner to match +the memory controller whose cgroups also persist as zombie until all charged +memory is uncharged. Also, the tracking of HugeTLB reservations is a bit more +complex compared to the tracking of HugeTLB faults, so it is significantly +harder to reparent reservations at offline time. diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index fbb111616705..bcc80269bb6a 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -188,6 +188,17 @@ cgroup v2 currently supports the following mount options. modified through remount from the init namespace. The mount option is ignored on non-init namespace mounts. + memory_recursiveprot + + Recursively apply memory.min and memory.low protection to + entire subtrees, without requiring explicit downward + propagation into leaf cgroups. This allows protecting entire + subtrees from one another, while retaining free competition + within those subtrees. This should have been the default + behavior but is a mount-option to avoid regressing setups + relying on the original semantics (e.g. specifying bogusly + high 'bypass' protection values at higher tree levels). + Organizing Processes and Threads -------------------------------- diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index ed73df5f1369..4d5a4fe22703 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3821,6 +3821,11 @@ before loading. See Documentation/admin-guide/blockdev/ramdisk.rst. + prot_virt= [S390] enable hosting protected virtual machines + isolated from the hypervisor (if hardware supports + that). + Format: + psi= [KNL] Enable or disable pressure stall information tracking. Format: diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 9e8470008227..290aa766629a 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -129,6 +129,9 @@ allowed to examine the unevictable lru (mlocked pages) for pages to compact. This should be used on systems where stalls for minor page faults are an acceptable trade for large contiguous free memory. Set to 0 to prevent compaction from moving pages that are unevictable. Default value is 1. +On CONFIG_PREEMPT_RT the default value is 0 in order to avoid a page fault, due +to compaction, which would block the task from becomming active until the fault +is resolved. dirty_background_bytes diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index be726986ff75..2adffb3f7914 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -73,6 +73,9 @@ File Mapping and Page Cache .. kernel-doc:: mm/truncate.c :export: +.. kernel-doc:: include/linux/pagemap.h + :internal: + Memory pools ============ diff --git a/Documentation/core-api/pin_user_pages.rst b/Documentation/core-api/pin_user_pages.rst index 1d490155ecd7..2e939ff10b86 100644 --- a/Documentation/core-api/pin_user_pages.rst +++ b/Documentation/core-api/pin_user_pages.rst @@ -52,8 +52,22 @@ Which flags are set by each wrapper For these pin_user_pages*() functions, FOLL_PIN is OR'd in with whatever gup flags the caller provides. The caller is required to pass in a non-null struct -pages* array, and the function then pin pages by incrementing each by a special -value. For now, that value is +1, just like get_user_pages*().:: +pages* array, and the function then pins pages by incrementing each by a special +value: GUP_PIN_COUNTING_BIAS. + +For huge pages (and in fact, any compound page of more than 2 pages), the +GUP_PIN_COUNTING_BIAS scheme is not used. Instead, an exact form of pin counting +is achieved, by using the 3rd struct page in the compound page. A new struct +page field, hpage_pinned_refcount, has been added in order to support this. + +This approach for compound pages avoids the counting upper limit problems that +are discussed below. Those limitations would have been aggravated severely by +huge pages, because each tail page adds a refcount to the head page. And in +fact, testing revealed that, without a separate hpage_pinned_refcount field, +page overflows were seen in some huge page stress tests. + +This also means that huge pages and compound pages (of order > 1) do not suffer +from the false positives problem that is mentioned below.:: Function -------- @@ -99,27 +113,6 @@ pages: This also leads to limitations: there are only 31-10==21 bits available for a counter that increments 10 bits at a time. -TODO: for 1GB and larger huge pages, this is cutting it close. That's because -when pin_user_pages() follows such pages, it increments the head page by "1" -(where "1" used to mean "+1" for get_user_pages(), but now means "+1024" for -pin_user_pages()) for each tail page. So if you have a 1GB huge page: - -* There are 256K (18 bits) worth of 4 KB tail pages. -* There are 21 bits available to count up via GUP_PIN_COUNTING_BIAS (that is, - 10 bits at a time) -* There are 21 - 18 == 3 bits available to count. Except that there aren't, - because you need to allow for a few normal get_page() calls on the head page, - as well. Fortunately, the approach of using addition, rather than "hard" - bitfields, within page->_refcount, allows for sharing these bits gracefully. - But we're still looking at about 8 references. - -This, however, is a missing feature more than anything else, because it's easily -solved by addressing an obvious inefficiency in the original get_user_pages() -approach of retrieving pages: stop treating all the pages as if they were -PAGE_SIZE. Retrieve huge pages as huge pages. The callers need to be aware of -this, so some work is required. Once that's in place, this limitation mostly -disappears from view, because there will be ample refcounting range available. - * Callers must specifically request "dma-pinned tracking of pages". In other words, just calling get_user_pages() will not suffice; a new set of functions, pin_user_page() and related, must be used. @@ -173,8 +166,8 @@ CASE 4: Pinning for struct page manipulation only ------------------------------------------------- Here, normal GUP calls are sufficient, so neither flag needs to be set. -page_dma_pinned(): the whole point of pinning -============================================= +page_maybe_dma_pinned(): the whole point of pinning +=================================================== The whole point of marking pages as "DMA-pinned" or "gup-pinned" is to be able to query, "is this page DMA-pinned?" That allows code such as page_mkclean() @@ -186,7 +179,7 @@ and debates (see the References at the end of this document). It's a TODO item here: fill in the details once that's worked out. Meanwhile, it's safe to say that having this available: :: - static inline bool page_dma_pinned(struct page *page) + static inline bool page_maybe_dma_pinned(struct page *page) ...is a prerequisite to solving the long-running gup+DMA problem. @@ -215,12 +208,42 @@ has the following new calls to exercise the new pin*() wrapper functions: You can monitor how many total dma-pinned pages have been acquired and released since the system was booted, via two new /proc/vmstat entries: :: - /proc/vmstat/nr_foll_pin_requested - /proc/vmstat/nr_foll_pin_requested + /proc/vmstat/nr_foll_pin_acquired + /proc/vmstat/nr_foll_pin_released -Those are both going to show zero, unless CONFIG_DEBUG_VM is set. This is -because there is a noticeable performance drop in unpin_user_page(), when they -are activated. +Under normal conditions, these two values will be equal unless there are any +long-term [R]DMA pins in place, or during pin/unpin transitions. + +* nr_foll_pin_acquired: This is the number of logical pins that have been + acquired since the system was powered on. For huge pages, the head page is + pinned once for each page (head page and each tail page) within the huge page. + This follows the same sort of behavior that get_user_pages() uses for huge + pages: the head page is refcounted once for each tail or head page in the huge + page, when get_user_pages() is applied to a huge page. + +* nr_foll_pin_released: The number of logical pins that have been released since + the system was powered on. Note that pages are released (unpinned) on a + PAGE_SIZE granularity, even if the original pin was applied to a huge page. + Becaused of the pin count behavior described above in "nr_foll_pin_acquired", + the accounting balances out, so that after doing this:: + + pin_user_pages(huge_page); + for (each page in huge_page) + unpin_user_page(page); + +...the following is expected:: + + nr_foll_pin_released == nr_foll_pin_acquired + +(...unless it was already out of balance due to a long-term RDMA pin being in +place.) + +Other diagnostics +================= + +dump_page() has been enhanced slightly, to handle these new counting fields, and +to better report on compound pages in general. Specifically, for compound pages +with order > 1, the exact (hpage_pinned_refcount) pincount is reported. References ========== @@ -228,5 +251,6 @@ References * `Some slow progress on get_user_pages() (Apr 2, 2019) `_ * `DMA and get_user_pages() (LPC: Dec 12, 2018) `_ * `The trouble with get_user_pages() (Apr 30, 2018) `_ +* `LWN kernel index: get_user_pages() `_ John Hubbard, October, 2019 diff --git a/Documentation/devicetree/bindings/dma/renesas,usb-dmac.txt b/Documentation/devicetree/bindings/dma/renesas,usb-dmac.txt index f1f95f678739..e8f6c42e80f2 100644 --- a/Documentation/devicetree/bindings/dma/renesas,usb-dmac.txt +++ b/Documentation/devicetree/bindings/dma/renesas,usb-dmac.txt @@ -16,6 +16,7 @@ Required Properties: - "renesas,r8a7794-usb-dmac" (R-Car E2) - "renesas,r8a7795-usb-dmac" (R-Car H3) - "renesas,r8a7796-usb-dmac" (R-Car M3-W) + - "renesas,r8a77961-usb-dmac" (R-Car M3-W+) - "renesas,r8a77965-usb-dmac" (R-Car M3-N) - "renesas,r8a77990-usb-dmac" (R-Car E3) - "renesas,r8a77995-usb-dmac" (R-Car D3) diff --git a/Documentation/devicetree/bindings/dma/socionext,uniphier-xdmac.yaml b/Documentation/devicetree/bindings/dma/socionext,uniphier-xdmac.yaml new file mode 100644 index 000000000000..86cfb599256e --- /dev/null +++ b/Documentation/devicetree/bindings/dma/socionext,uniphier-xdmac.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/socionext,uniphier-xdmac.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Socionext UniPhier external DMA controller + +description: | + This describes the devicetree bindings for an external DMA engine to perform + memory-to-memory or peripheral-to-memory data transfer capable of supporting + 16 channels, implemented in Socionext UniPhier SoCs. + +maintainers: + - Kunihiko Hayashi + +allOf: + - $ref: "dma-controller.yaml#" + +properties: + compatible: + const: socionext,uniphier-xdmac + + reg: + items: + - description: XDMAC base register region (offset and length) + - description: XDMAC extension register region (offset and length) + + interrupts: + maxItems: 1 + + "#dma-cells": + const: 2 + description: | + DMA request from clients consists of 2 cells: + 1. Channel index + 2. Transfer request factor number, If no transfer factor, use 0. + The number is SoC-specific, and this should be specified with + relation to the device to use the DMA controller. + + dma-channels: + minimum: 1 + maximum: 16 + +additionalProperties: false + +required: + - compatible + - reg + - interrupts + - "#dma-cells" + +examples: + - | + xdmac: dma-controller@5fc10000 { + compatible = "socionext,uniphier-xdmac"; + reg = <0x5fc10000 0x1000>, <0x5fc20000 0x800>; + interrupts = <0 188 4>; + #dma-cells = <2>; + dma-channels = <16>; + }; + +... diff --git a/Documentation/devicetree/bindings/dma/ti-edma.txt b/Documentation/devicetree/bindings/dma/ti-edma.txt index 0e1398f93aa2..29fcd37082e8 100644 --- a/Documentation/devicetree/bindings/dma/ti-edma.txt +++ b/Documentation/devicetree/bindings/dma/ti-edma.txt @@ -180,7 +180,7 @@ edma1_tptc0: tptc@27b0000 { }; edma1_tptc1: tptc@27b8000 { - compatible = "ti, k2g-edma3-tptc", "ti,edma3-tptc"; + compatible = "ti,k2g-edma3-tptc", "ti,edma3-tptc"; reg = <0x027b8000 0x400>; power-domains = <&k2g_pds 0x4f>; }; diff --git a/Documentation/devicetree/bindings/dma/ti/k3-udma.yaml b/Documentation/devicetree/bindings/dma/ti/k3-udma.yaml index 34780d7535b8..39ea05e6e5ff 100644 --- a/Documentation/devicetree/bindings/dma/ti/k3-udma.yaml +++ b/Documentation/devicetree/bindings/dma/ti/k3-udma.yaml @@ -45,7 +45,8 @@ allOf: properties: "#dma-cells": - const: 1 + minimum: 1 + maximum: 2 description: | The cell is the PSI-L thread ID of the remote (to UDMAP) end. Valid ranges for thread ID depends on the data movement direction: @@ -55,6 +56,8 @@ properties: Please refer to the device documentation for the PSI-L thread map and also the PSI-L peripheral chapter for the correct thread ID. + When #dma-cells is 2, the second parameter is the channel ATYPE. + compatible: enum: - ti,am654-navss-main-udmap @@ -131,6 +134,20 @@ required: - ti,sci-rm-range-rchan - ti,sci-rm-range-rflow +if: + properties: + "#dma-cells": + const: 2 +then: + properties: + ti,udma-atype: + description: ATYPE value which should be used by non slave channels + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + + required: + - ti,udma-atype + examples: - |+ cbass_main { diff --git a/Documentation/devicetree/bindings/i2c/brcm,brcmstb-i2c.yaml b/Documentation/devicetree/bindings/i2c/brcm,brcmstb-i2c.yaml new file mode 100644 index 000000000000..edbca2476128 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/brcm,brcmstb-i2c.yaml @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i2c/brcm,brcmstb-i2c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom STB BSC IIC Master Controller + +maintainers: + - Kamal Dasu + +allOf: + - $ref: /schemas/i2c/i2c-controller.yaml# + +properties: + compatible: + enum: + - brcm,bcm2711-hdmi-i2c + - brcm,brcmstb-i2c + - brcm,brcmper-i2c + + reg: + minItems: 1 + maxItems: 2 + items: + - description: BSC register range + - description: Auto-I2C register range + + reg-names: + items: + - const: bsc + - const: auto-i2c + + interrupts: + maxItems: 1 + + interrupt-names: + maxItems: 1 + + clock-frequency: + enum: + - 46875 + - 50000 + - 93750 + - 97500 + - 187500 + - 200000 + - 375000 + - 390000 + +required: + - compatible + - reg + - clock-frequency + +unevaluatedProperties: false + +if: + properties: + compatible: + contains: + enum: + - brcm,bcm2711-hdmi-i2c + +then: + properties: + reg: + minItems: 2 + + required: + - reg-names + +else: + properties: + reg: + maxItems: 1 + +examples: + - | + bsca: i2c@f0406200 { + clock-frequency = <390000>; + compatible = "brcm,brcmstb-i2c"; + interrupt-parent = <&irq0_intc>; + reg = <0xf0406200 0x58>; + interrupts = <0x18>; + interrupt-names = "upg_bsca"; + }; + + - | + ddc0: i2c@7ef04500 { + compatible = "brcm,bcm2711-hdmi-i2c"; + reg = <0x7ef04500 0x100>, <0x7ef00b00 0x300>; + reg-names = "bsc", "auto-i2c"; + clock-frequency = <390000>; + }; + +... diff --git a/Documentation/devicetree/bindings/i2c/i2c-at91.txt b/Documentation/devicetree/bindings/i2c/i2c-at91.txt index d4bad86107b8..96c914e048f5 100644 --- a/Documentation/devicetree/bindings/i2c/i2c-at91.txt +++ b/Documentation/devicetree/bindings/i2c/i2c-at91.txt @@ -28,8 +28,13 @@ Optional properties: "atmel,sama5d4-i2c", "atmel,sama5d2-i2c", "microchip,sam9x60-i2c". +- scl-gpios: specify the gpio related to SCL pin +- sda-gpios: specify the gpio related to SDA pin +- pinctrl: add extra pinctrl to configure i2c pins to gpio function for i2c + bus recovery, call it "gpio" state - Child nodes conforming to i2c bus binding + Examples : i2c0: i2c@fff84000 { @@ -64,6 +69,11 @@ i2c0: i2c@f8034600 { clocks = <&flx0>; atmel,fifo-size = <16>; i2c-sda-hold-time-ns = <336>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c0>; + pinctrl-1 = <&pinctrl_i2c0_gpio>; + sda-gpios = <&pioA 30 GPIO_ACTIVE_HIGH>; + scl-gpios = <&pioA 31 GPIO_ACTIVE_HIGH>; wm8731: wm8731@1a { compatible = "wm8731"; diff --git a/Documentation/devicetree/bindings/i2c/i2c-brcmstb.txt b/Documentation/devicetree/bindings/i2c/i2c-brcmstb.txt deleted file mode 100644 index 0380609b177a..000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-brcmstb.txt +++ /dev/null @@ -1,26 +0,0 @@ -Broadcom stb bsc iic master controller - -Required properties: - -- compatible: should be "brcm,brcmstb-i2c" or "brcm,brcmper-i2c" -- clock-frequency: 32-bit decimal value of iic master clock freqency in Hz - valid values are 375000, 390000, 187500, 200000 - 93750, 97500, 46875 and 50000 -- reg: specifies the base physical address and size of the registers - -Optional properties : - -- interrupts: specifies the interrupt number, the irq line to be used -- interrupt-names: Interrupt name string - -Example: - -bsca: i2c@f0406200 { - clock-frequency = <390000>; - compatible = "brcm,brcmstb-i2c"; - interrupt-parent = <&irq0_intc>; - reg = <0xf0406200 0x58>; - interrupts = <0x18>; - interrupt-names = "upg_bsca"; -}; - diff --git a/Documentation/devicetree/bindings/mtd/brcm,brcmnand.txt b/Documentation/devicetree/bindings/mtd/brcm,brcmnand.txt index 82156dc8f304..05651a654c66 100644 --- a/Documentation/devicetree/bindings/mtd/brcm,brcmnand.txt +++ b/Documentation/devicetree/bindings/mtd/brcm,brcmnand.txt @@ -35,11 +35,11 @@ Required properties: (optional) NAND flash cache range (if at non-standard offset) - reg-names : a list of the names corresponding to the previous register ranges. Should contain "nand" and (optionally) - "flash-dma" and/or "nand-cache". -- interrupts : The NAND CTLRDY interrupt and (if Flash DMA is available) - FLASH_DMA_DONE -- interrupt-names : May be "nand_ctlrdy" or "flash_dma_done", if broken out as - individual interrupts. + "flash-dma" or "flash-edu" and/or "nand-cache". +- interrupts : The NAND CTLRDY interrupt, (if Flash DMA is available) + FLASH_DMA_DONE and if EDU is avaialble and used FLASH_EDU_DONE +- interrupt-names : May be "nand_ctlrdy" or "flash_dma_done" or "flash_edu_done", + if broken out as individual interrupts. May be "nand", if the SoC has the individual NAND interrupts multiplexed behind another custom piece of hardware diff --git a/Documentation/devicetree/bindings/mtd/nand-macronix.txt b/Documentation/devicetree/bindings/mtd/nand-macronix.txt new file mode 100644 index 000000000000..ffab28a2c4d1 --- /dev/null +++ b/Documentation/devicetree/bindings/mtd/nand-macronix.txt @@ -0,0 +1,27 @@ +Macronix NANDs Device Tree Bindings +----------------------------------- + +Macronix NANDs support randomizer operation for scrambling user data, +which can be enabled with a SET_FEATURE. The penalty when using the +randomizer are subpage accesses prohibited and more time period needed +for program operation, i.e., tPROG 300us to 340us (randomizer enabled). +Enabling the randomizer is a one time persistent and non reversible +operation. + +For more high-reliability concern, if subpage write is not available +with hardware ECC and not enabled at UBI level, then enabling the +randomizer is recommended by default by adding a new specific property +in children nodes. + +Required NAND chip properties in children mode: +- randomizer enable: should be "mxic,enable-randomizer-otp" + +Example: + + nand: nand-controller@unit-address { + + nand@0 { + reg = <0>; + mxic,enable-randomizer-otp; + }; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/atmel,at91-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/atmel,at91-pinctrl.txt index eb39f5051159..e8abbdad7b5d 100644 --- a/Documentation/devicetree/bindings/pinctrl/atmel,at91-pinctrl.txt +++ b/Documentation/devicetree/bindings/pinctrl/atmel,at91-pinctrl.txt @@ -38,7 +38,7 @@ Bank: 3 (A, B and C) 0xffffffff 0x7fff3ccf /* pioB */ 0xffffffff 0x007fffff /* pioC */ -For each peripheral/bank we will descibe in a u32 if a pin can be +For each peripheral/bank we will describe in a u32 if a pin can be configured in it by putting 1 to the pin bit (1 << pin) Let's take the pioA on peripheral B diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mm-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mm-pinctrl.txt deleted file mode 100644 index e4e01c05cf83..000000000000 --- a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mm-pinctrl.txt +++ /dev/null @@ -1,36 +0,0 @@ -* Freescale IMX8MM IOMUX Controller - -Please refer to fsl,imx-pinctrl.txt and pinctrl-bindings.txt in this directory -for common binding part and usage. - -Required properties: -- compatible: "fsl,imx8mm-iomuxc" -- reg: should contain the base physical address and size of the iomuxc - registers. - -Required properties in sub-nodes: -- fsl,pins: each entry consists of 6 integers and represents the mux and config - setting for one pin. The first 5 integers are specified using a PIN_FUNC_ID macro, which can be found in - . The last integer CONFIG is - the pad setting value like pull-up on this pin. Please refer to i.MX8M Mini - Reference Manual for detailed CONFIG settings. - -Examples: - -&uart1 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_uart1>; -}; - -iomuxc: pinctrl@30330000 { - compatible = "fsl,imx8mm-iomuxc"; - reg = <0x0 0x30330000 0x0 0x10000>; - - pinctrl_uart1: uart1grp { - fsl,pins = < - MX8MM_IOMUXC_UART1_RXD_UART1_DCE_RX 0x140 - MX8MM_IOMUXC_UART1_TXD_UART1_DCE_TX 0x140 - >; - }; -}; diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mm-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mm-pinctrl.yaml new file mode 100644 index 000000000000..d98a3866add8 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mm-pinctrl.yaml @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/fsl,imx8mm-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Freescale IMX8MM IOMUX Controller + +maintainers: + - Anson Huang + +description: + Please refer to fsl,imx-pinctrl.txt and pinctrl-bindings.txt in this directory + for common binding part and usage. + +properties: + compatible: + const: fsl,imx8mm-iomuxc + + reg: + maxItems: 1 + +# Client device subnode's properties +patternProperties: + 'grp$': + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + + properties: + fsl,pins: + description: + each entry consists of 6 integers and represents the mux and config + setting for one pin. The first 5 integers are specified using a PIN_FUNC_ID macro, which can + be found in . The last + integer CONFIG is the pad setting value like pull-up on this pin. Please + refer to i.MX8M Mini Reference Manual for detailed CONFIG settings. + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32-matrix + - items: + items: + - description: | + "mux_reg" indicates the offset of mux register. + - description: | + "conf_reg" indicates the offset of pad configuration register. + - description: | + "input_reg" indicates the offset of select input register. + - description: | + "mux_val" indicates the mux value to be applied. + - description: | + "input_val" indicates the select input value to be applied. + - description: | + "pad_setting" indicates the pad configuration value to be applied. + + required: + - fsl,pins + + additionalProperties: false + +required: + - compatible + - reg + +additionalProperties: false + +examples: + # Pinmux controller node + - | + iomuxc: pinctrl@30330000 { + compatible = "fsl,imx8mm-iomuxc"; + reg = <0x30330000 0x10000>; + + pinctrl_uart2: uart2grp { + fsl,pins = + <0x23C 0x4A4 0x4FC 0x0 0x0 0x140>, + <0x240 0x4A8 0x000 0x0 0x0 0x140>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mn-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mn-pinctrl.txt deleted file mode 100644 index 330716c971b9..000000000000 --- a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mn-pinctrl.txt +++ /dev/null @@ -1,39 +0,0 @@ -* Freescale IMX8MN IOMUX Controller - -Please refer to fsl,imx-pinctrl.txt and pinctrl-bindings.txt in this directory -for common binding part and usage. - -Required properties: -- compatible: "fsl,imx8mn-iomuxc" -- reg: should contain the base physical address and size of the iomuxc - registers. - -Required properties in sub-nodes: -- fsl,pins: each entry consists of 6 integers and represents the mux and config - setting for one pin. The first 5 integers are specified using a PIN_FUNC_ID macro, which can be found in - . The last integer CONFIG is - the pad setting value like pull-up on this pin. Please refer to i.MX8M Nano - Reference Manual for detailed CONFIG settings. - -Examples: - -&uart1 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_uart1>; -}; - -iomuxc: pinctrl@30330000 { - compatible = "fsl,imx8mn-iomuxc"; - reg = <0x0 0x30330000 0x0 0x10000>; - - pinctrl_uart1: uart1grp { - fsl,pins = < - MX8MN_IOMUXC_UART1_RXD_UART1_DCE_RX 0x140 - MX8MN_IOMUXC_UART1_TXD_UART1_DCE_TX 0x140 - MX8MN_IOMUXC_UART3_RXD_UART1_DCE_CTS_B 0x140 - MX8MN_IOMUXC_UART3_TXD_UART1_DCE_RTS_B 0x140 - MX8MN_IOMUXC_SD1_DATA4_GPIO2_IO6 0x19 - >; - }; -}; diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mn-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mn-pinctrl.yaml new file mode 100644 index 000000000000..b9aa180e07e4 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mn-pinctrl.yaml @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/fsl,imx8mn-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Freescale IMX8MN IOMUX Controller + +maintainers: + - Anson Huang + +description: + Please refer to fsl,imx-pinctrl.txt and pinctrl-bindings.txt in this directory + for common binding part and usage. + +properties: + compatible: + const: fsl,imx8mn-iomuxc + + reg: + maxItems: 1 + +# Client device subnode's properties +patternProperties: + 'grp$': + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + + properties: + fsl,pins: + description: + each entry consists of 6 integers and represents the mux and config + setting for one pin. The first 5 integers are specified using a PIN_FUNC_ID macro, which can + be found in . The last + integer CONFIG is the pad setting value like pull-up on this pin. Please + refer to i.MX8M Nano Reference Manual for detailed CONFIG settings. + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32-matrix + - items: + items: + - description: | + "mux_reg" indicates the offset of mux register. + - description: | + "conf_reg" indicates the offset of pad configuration register. + - description: | + "input_reg" indicates the offset of select input register. + - description: | + "mux_val" indicates the mux value to be applied. + - description: | + "input_val" indicates the select input value to be applied. + - description: | + "pad_setting" indicates the pad configuration value to be applied. + + required: + - fsl,pins + + additionalProperties: false + +required: + - compatible + - reg + +additionalProperties: false + +examples: + # Pinmux controller node + - | + iomuxc: pinctrl@30330000 { + compatible = "fsl,imx8mn-iomuxc"; + reg = <0x30330000 0x10000>; + + pinctrl_uart2: uart2grp { + fsl,pins = + <0x23C 0x4A4 0x4FC 0x0 0x0 0x140>, + <0x240 0x4A8 0x000 0x0 0x0 0x140>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mp-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mp-pinctrl.yaml index 2e31e120395e..6297e78418cf 100644 --- a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mp-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mp-pinctrl.yaml @@ -30,8 +30,6 @@ patternProperties: properties: fsl,pins: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array description: each entry consists of 6 integers and represents the mux and config setting for one pin. The first 5 integers . The last integer CONFIG is the pad setting value like pull-up on this pin. Please refer to i.MX8M Plus Reference Manual for detailed CONFIG settings. + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32-matrix + - items: + items: + - description: | + "mux_reg" indicates the offset of mux register. + - description: | + "conf_reg" indicates the offset of pad configuration register. + - description: | + "input_reg" indicates the offset of select input register. + - description: | + "mux_val" indicates the mux value to be applied. + - description: | + "input_val" indicates the select input value to be applied. + - description: | + "pad_setting" indicates the pad configuration value to be applied. required: - fsl,pins @@ -59,10 +73,9 @@ examples: reg = <0x30330000 0x10000>; pinctrl_uart2: uart2grp { - fsl,pins = < - 0x228 0x488 0x5F0 0x0 0x6 0x49 - 0x228 0x488 0x000 0x0 0x0 0x49 - >; + fsl,pins = + <0x228 0x488 0x5F0 0x0 0x6 0x49>, + <0x228 0x488 0x000 0x0 0x0 0x49>; }; }; diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mq-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mq-pinctrl.txt deleted file mode 100644 index 66de75090458..000000000000 --- a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mq-pinctrl.txt +++ /dev/null @@ -1,36 +0,0 @@ -* Freescale IMX8MQ IOMUX Controller - -Please refer to fsl,imx-pinctrl.txt and pinctrl-bindings.txt in this directory -for common binding part and usage. - -Required properties: -- compatible: "fsl,imx8mq-iomuxc" -- reg: should contain the base physical address and size of the iomuxc - registers. - -Required properties in sub-nodes: -- fsl,pins: each entry consists of 6 integers and represents the mux and config - setting for one pin. The first 5 integers are specified using a PIN_FUNC_ID macro, which can be found in - imx8mq-pinfunc.h under device tree source folder. The last integer CONFIG is - the pad setting value like pull-up on this pin. Please refer to i.MX8M Quad - Reference Manual for detailed CONFIG settings. - -Examples: - -&uart1 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_uart1>; -}; - -iomuxc: pinctrl@30330000 { - compatible = "fsl,imx8mq-iomuxc"; - reg = <0x0 0x30330000 0x0 0x10000>; - - pinctrl_uart1: uart1grp { - fsl,pins = < - MX8MQ_IOMUXC_UART1_RXD_UART1_DCE_RX 0x49 - MX8MQ_IOMUXC_UART1_TXD_UART1_DCE_TX 0x49 - >; - }; -}; diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx8mq-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mq-pinctrl.yaml new file mode 100644 index 000000000000..b30c704fcfa1 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx8mq-pinctrl.yaml @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/fsl,imx8mq-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Freescale IMX8MQ IOMUX Controller + +maintainers: + - Anson Huang + +description: + Please refer to fsl,imx-pinctrl.txt and pinctrl-bindings.txt in this directory + for common binding part and usage. + +properties: + compatible: + const: fsl,imx8mq-iomuxc + + reg: + maxItems: 1 + +# Client device subnode's properties +patternProperties: + 'grp$': + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + + properties: + fsl,pins: + description: + each entry consists of 6 integers and represents the mux and config + setting for one pin. The first 5 integers are specified using a PIN_FUNC_ID macro, which can + be found in . The last + integer CONFIG is the pad setting value like pull-up on this pin. Please + refer to i.MX8M Quad Reference Manual for detailed CONFIG settings. + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32-matrix + - items: + items: + - description: | + "mux_reg" indicates the offset of mux register. + - description: | + "conf_reg" indicates the offset of pad configuration register. + - description: | + "input_reg" indicates the offset of select input register. + - description: | + "mux_val" indicates the mux value to be applied. + - description: | + "input_val" indicates the select input value to be applied. + - description: | + "pad_setting" indicates the pad configuration value to be applied. + + required: + - fsl,pins + + additionalProperties: false + +required: + - compatible + - reg + +additionalProperties: false + +examples: + # Pinmux controller node + - | + iomuxc: pinctrl@30330000 { + compatible = "fsl,imx8mq-iomuxc"; + reg = <0x30330000 0x10000>; + + pinctrl_uart1: uart1grp { + fsl,pins = + <0x234 0x49C 0x4F4 0x0 0x0 0x49>, + <0x238 0x4A0 0x4F4 0x0 0x0 0x49>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,ipq6018-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,ipq6018-pinctrl.yaml new file mode 100644 index 000000000000..63d1cfe86c6e --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,ipq6018-pinctrl.yaml @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,ipq6018-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. IPQ6018 TLMM block + +maintainers: + - Sricharan R + +description: | + This binding describes the Top Level Mode Multiplexer block found in the + IPQ6018 platform. + +properties: + compatible: + const: qcom,ipq6018-pinctrl + + reg: + maxItems: 1 + + interrupts: + description: Specifies the TLMM summary IRQ + maxItems: 1 + + interrupt-controller: true + + '#interrupt-cells': + description: + Specifies the PIN numbers and Flags, as defined in defined in + include/dt-bindings/interrupt-controller/irq.h + const: 2 + + gpio-controller: true + + '#gpio-cells': + description: Specifying the pin number and flags, as defined in + include/dt-bindings/gpio/gpio.h + const: 2 + + gpio-ranges: + maxItems: 1 + +#PIN CONFIGURATION NODES +patternProperties: + '-pinmux$': + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + allOf: + - $ref: "/schemas/pinctrl/pincfg-node.yaml" + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + oneOf: + - pattern: "^gpio([1-9]|[1-7][0-9]|80)$" + - enum: [ sdc1_clk, sdc1_cmd, sdc1_data, sdc2_clk, sdc2_cmd, + sdc2_data, qdsd_cmd, qdsd_data0, qdsd_data1, qdsd_data2, + qdsd_data3 ] + minItems: 1 + maxItems: 4 + + function: + description: + Specify the alternative function to be configured for the specified + pins. + enum: [ adsp_ext, alsp_int, atest_bbrx0, atest_bbrx1, atest_char, + atest_char0, atest_char1, atest_char2, atest_char3, atest_combodac, + atest_gpsadc0, atest_gpsadc1, atest_tsens, atest_wlan0, + atest_wlan1, backlight_en, bimc_dte0, bimc_dte1, blsp1_i2c, + blsp2_i2c, blsp3_i2c, blsp4_i2c, blsp5_i2c, blsp6_i2c, blsp1_spi, + blsp1_spi_cs1, blsp1_spi_cs2, blsp1_spi_cs3, blsp2_spi, + blsp2_spi_cs1, blsp2_spi_cs2, blsp2_spi_cs3, blsp3_spi, + blsp3_spi_cs1, blsp3_spi_cs2, blsp3_spi_cs3, blsp4_spi, blsp5_spi, + blsp6_spi, blsp1_uart, blsp2_uart, blsp1_uim, blsp2_uim, cam1_rst, + cam1_standby, cam_mclk0, cam_mclk1, cci_async, cci_i2c, cci_timer0, + cci_timer1, cci_timer2, cdc_pdm0, codec_mad, dbg_out, display_5v, + dmic0_clk, dmic0_data, dsi_rst, ebi0_wrcdc, euro_us, ext_lpass, + flash_strobe, gcc_gp1_clk_a, gcc_gp1_clk_b, gcc_gp2_clk_a, + gcc_gp2_clk_b, gcc_gp3_clk_a, gcc_gp3_clk_b, gpio, gsm0_tx0, + gsm0_tx1, gsm1_tx0, gsm1_tx1, gyro_accl, kpsns0, kpsns1, kpsns2, + ldo_en, ldo_update, mag_int, mdp_vsync, modem_tsync, m_voc, + nav_pps, nav_tsync, pa_indicator, pbs0, pbs1, pbs2, pri_mi2s, + pri_mi2s_ws, prng_rosc, pwr_crypto_enabled_a, pwr_crypto_enabled_b, + pwr_modem_enabled_a, pwr_modem_enabled_b, pwr_nav_enabled_a, + pwr_nav_enabled_b, qdss_ctitrig_in_a0, qdss_ctitrig_in_a1, + qdss_ctitrig_in_b0, qdss_ctitrig_in_b1, qdss_ctitrig_out_a0, + qdss_ctitrig_out_a1, qdss_ctitrig_out_b0, qdss_ctitrig_out_b1, + qdss_traceclk_a, qdss_traceclk_b, qdss_tracectl_a, qdss_tracectl_b, + qdss_tracedata_a, qdss_tracedata_b, reset_n, sd_card, sd_write, + sec_mi2s, smb_int, ssbi_wtr0, ssbi_wtr1, uim1, uim2, uim3, + uim_batt, wcss_bt, wcss_fm, wcss_wlan, webcam1_rst ] + + drive-strength: + enum: [2, 4, 6, 8, 10, 12, 14, 16] + default: 2 + description: + Selects the drive strength for the specified pins, in mA. + + bias-pull-down: true + + bias-pull-up: true + + bias-disable: true + + output-high: true + + output-low: true + + required: + - pins + - function + + additionalProperties: false + +required: + - compatible + - reg + - interrupts + - interrupt-controller + - '#interrupt-cells' + - gpio-controller + - '#gpio-cells' + - gpio-ranges + +additionalProperties: false + +examples: + - | + #include + tlmm: pinctrl@1000000 { + compatible = "qcom,ipq6018-pinctrl"; + reg = <0x01000000 0x300000>; + interrupts = ; + interrupt-controller; + #interrupt-cells = <2>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&tlmm 0 80>; + + serial3-pinmux { + pins = "gpio44", "gpio45"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; diff --git a/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml b/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml new file mode 100644 index 000000000000..084960a8f17a --- /dev/null +++ b/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml @@ -0,0 +1,324 @@ +# SPDX-License-Identifier: (GPL-2.0-only or BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/remoteproc/ti,omap-remoteproc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: OMAP4+ Remoteproc Devices + +maintainers: + - Suman Anna + +description: + The OMAP family of SoCs usually have one or more slave processor sub-systems + that are used to offload some of the processor-intensive tasks, or to manage + other hardware accelerators, for achieving various system level goals. + + The processor cores in the sub-system are usually behind an IOMMU, and may + contain additional sub-modules like Internal RAM and/or ROMs, L1 and/or L2 + caches, an Interrupt Controller, a Cache Controller etc. + + The OMAP SoCs usually have a DSP processor sub-system and/or an IPU processor + sub-system. The DSP processor sub-system can contain any of the TI's C64x, + C66x or C67x family of DSP cores as the main execution unit. The IPU processor + sub-system usually contains either a Dual-Core Cortex-M3 or Dual-Core + Cortex-M4 processors. + + Each remote processor sub-system is represented as a single DT node. Each node + has a number of required or optional properties that enable the OS running on + the host processor (MPU) to perform the device management of the remote + processor and to communicate with the remote processor. The various properties + can be classified as constant or variable. The constant properties are + dictated by the SoC and does not change from one board to another having the + same SoC. Examples of constant properties include 'iommus', 'reg'. The + variable properties are dictated by the system integration aspects such as + memory on the board, or configuration used within the corresponding firmware + image. Examples of variable properties include 'mboxes', 'memory-region', + 'timers', 'watchdog-timers' etc. + +properties: + compatible: + enum: + - ti,omap4-dsp + - ti,omap5-dsp + - ti,dra7-dsp + - ti,omap4-ipu + - ti,omap5-ipu + - ti,dra7-ipu + + iommus: + minItems: 1 + maxItems: 2 + description: | + phandles to OMAP IOMMU nodes, that need to be programmed + for this remote processor to access any external RAM memory or + other peripheral device address spaces. This property usually + has only a single phandle. Multiple phandles are used only in + cases where the sub-system has different ports for different + sub-modules within the processor sub-system (eg: DRA7 DSPs), + and need the same programming in both the MMUs. + + mboxes: + minItems: 1 + maxItems: 2 + description: | + OMAP Mailbox specifier denoting the sub-mailbox, to be used for + communication with the remote processor. The specifier format is + as per the bindings, + Documentation/devicetree/bindings/mailbox/omap-mailbox.txt + This property should match with the sub-mailbox node used in + the firmware image. + + clocks: + description: | + Main functional clock for the remote processor + + resets: + description: | + Reset handles for the remote processor + + firmware-name: + description: | + Default name of the firmware to load to the remote processor. + +# Optional properties: +# -------------------- +# Some of these properties are mandatory on some SoCs, and some are optional +# depending on the configuration of the firmware image to be executed on the +# remote processor. The conditions are mentioned for each property. +# +# The following are the optional properties: + + memory-region: + $ref: /schemas/types.yaml#/definitions/phandle + description: | + phandle to the reserved memory node to be associated + with the remoteproc device. The reserved memory node + can be a CMA memory node, and should be defined as + per the bindings, + Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt + + reg: + description: | + Address space for any remoteproc memories present on + the SoC. Should contain an entry for each value in + 'reg-names'. These are mandatory for all DSP and IPU + processors that have them (OMAP4/OMAP5 DSPs do not have + any RAMs) + + reg-names: + description: | + Required names for each of the address spaces defined in + the 'reg' property. Expects the names from the following + list, in the specified order, each representing the corresponding + internal RAM memory region. + minItems: 1 + maxItems: 3 + items: + - const: l2ram + - const: l1pram + - const: l1dram + + ti,bootreg: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + Should be a triple of the phandle to the System Control + Configuration region that contains the boot address + register, the register offset of the boot address + register within the System Control module, and the bit + shift within the register. This property is required for + all the DSP instances on OMAP4, OMAP5 and DRA7xx SoCs. + + ti,autosuspend-delay-ms: + description: | + Custom autosuspend delay for the remoteproc in milliseconds. + Recommended values is preferable to be in the order of couple + of seconds. A negative value can also be used to disable the + autosuspend behavior. + + ti,timers: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + One or more phandles to OMAP DMTimer nodes, that serve + as System/Tick timers for the OS running on the remote + processors. This will usually be a single timer if the + processor sub-system is running in SMP mode, or one per + core in the processor sub-system. This can also be used + to reserve specific timers to be dedicated to the + remote processors. + + This property is mandatory on remote processors requiring + external tick wakeup, and to support Power Management + features. The timers to be used should match with the + timers used in the firmware image. + + ti,watchdog-timers: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + One or more phandles to OMAP DMTimer nodes, used to + serve as Watchdog timers for the processor cores. This + will usually be one per executing processor core, even + if the processor sub-system is running a SMP OS. + + The timers to be used should match with the watchdog + timers used in the firmware image. + +if: + properties: + compatible: + enum: + - ti,dra7-dsp +then: + properties: + reg: + minItems: 3 + maxItems: 3 + required: + - reg + - reg-names + - ti,bootreg + +else: + if: + properties: + compatible: + enum: + - ti,omap4-ipu + - ti,omap5-ipu + - ti,dra7-ipu + then: + properties: + reg: + minItems: 1 + maxItems: 1 + ti,bootreg: false + required: + - reg + - reg-names + + else: + properties: + reg: false + required: + - ti,bootreg + +required: + - compatible + - iommus + - mboxes + - clocks + - resets + - firmware-name + +additionalProperties: false + +examples: + - | + + //Example 1: OMAP4 DSP + + /* DSP Reserved Memory node */ + #include + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + + dsp_memory_region: dsp-memory@98000000 { + compatible = "shared-dma-pool"; + reg = <0x98000000 0x800000>; + reusable; + }; + }; + + /* DSP node */ + ocp { + dsp: dsp { + compatible = "ti,omap4-dsp"; + ti,bootreg = <&scm_conf 0x304 0>; + iommus = <&mmu_dsp>; + mboxes = <&mailbox &mbox_dsp>; + memory-region = <&dsp_memory_region>; + ti,timers = <&timer5>; + ti,watchdog-timers = <&timer6>; + clocks = <&tesla_clkctrl OMAP4_DSP_CLKCTRL 0>; + resets = <&prm_tesla 0>, <&prm_tesla 1>; + firmware-name = "omap4-dsp-fw.xe64T"; + }; + }; + + - |+ + + //Example 2: OMAP5 IPU + + /* IPU Reserved Memory node */ + #include + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + + ipu_memory_region: ipu-memory@95800000 { + compatible = "shared-dma-pool"; + reg = <0 0x95800000 0 0x3800000>; + reusable; + }; + }; + + /* IPU node */ + ocp { + #address-cells = <1>; + #size-cells = <1>; + + ipu: ipu@55020000 { + compatible = "ti,omap5-ipu"; + reg = <0x55020000 0x10000>; + reg-names = "l2ram"; + iommus = <&mmu_ipu>; + mboxes = <&mailbox &mbox_ipu>; + memory-region = <&ipu_memory_region>; + ti,timers = <&timer3>, <&timer4>; + ti,watchdog-timers = <&timer9>, <&timer11>; + clocks = <&ipu_clkctrl OMAP5_MMU_IPU_CLKCTRL 0>; + resets = <&prm_core 2>; + firmware-name = "omap5-ipu-fw.xem4"; + }; + }; + + - |+ + + //Example 3: DRA7xx/AM57xx DSP + + /* DSP1 Reserved Memory node */ + #include + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + + dsp1_memory_region: dsp1-memory@99000000 { + compatible = "shared-dma-pool"; + reg = <0x0 0x99000000 0x0 0x4000000>; + reusable; + }; + }; + + /* DSP1 node */ + ocp { + #address-cells = <1>; + #size-cells = <1>; + + dsp1: dsp@40800000 { + compatible = "ti,dra7-dsp"; + reg = <0x40800000 0x48000>, + <0x40e00000 0x8000>, + <0x40f00000 0x8000>; + reg-names = "l2ram", "l1pram", "l1dram"; + ti,bootreg = <&scm_conf 0x55c 0>; + iommus = <&mmu0_dsp1>, <&mmu1_dsp1>; + mboxes = <&mailbox5 &mbox_dsp1_ipc3x>; + memory-region = <&dsp1_memory_region>; + ti,timers = <&timer5>; + ti,watchdog-timers = <&timer10>; + resets = <&prm_dsp1 0>; + clocks = <&dsp1_clkctrl DRA7_DSP1_MMU0_DSP1_CLKCTRL 0>; + firmware-name = "dra7-dsp1-fw.xe66"; + }; + }; diff --git a/Documentation/devicetree/bindings/sound/amlogic,aiu.yaml b/Documentation/devicetree/bindings/sound/amlogic,aiu.yaml new file mode 100644 index 000000000000..a61bccf915d8 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/amlogic,aiu.yaml @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/amlogic,aiu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Amlogic AIU audio output controller + +maintainers: + - Jerome Brunet + +properties: + $nodename: + pattern: "^audio-controller@.*" + + "#sound-dai-cells": + const: 2 + + compatible: + items: + - enum: + - amlogic,aiu-gxbb + - amlogic,aiu-gxl + - amlogic,aiu-meson8 + - amlogic,aiu-meson8b + - const: + amlogic,aiu + + clocks: + items: + - description: AIU peripheral clock + - description: I2S peripheral clock + - description: I2S output clock + - description: I2S master clock + - description: I2S mixer clock + - description: SPDIF peripheral clock + - description: SPDIF output clock + - description: SPDIF master clock + - description: SPDIF master clock multiplexer + + clock-names: + items: + - const: pclk + - const: i2s_pclk + - const: i2s_aoclk + - const: i2s_mclk + - const: i2s_mixer + - const: spdif_pclk + - const: spdif_aoclk + - const: spdif_mclk + - const: spdif_mclk_sel + + interrupts: + items: + - description: I2S interrupt line + - description: SPDIF interrupt line + + interrupt-names: + items: + - const: i2s + - const: spdif + + reg: + maxItems: 1 + + resets: + maxItems: 1 + +required: + - "#sound-dai-cells" + - compatible + - clocks + - clock-names + - interrupts + - interrupt-names + - reg + - resets + +examples: + - | + #include + #include + #include + #include + + aiu: audio-controller@5400 { + compatible = "amlogic,aiu-gxl", "amlogic,aiu"; + #sound-dai-cells = <2>; + reg = <0x0 0x5400 0x0 0x2ac>; + interrupts = , + ; + interrupt-names = "i2s", "spdif"; + clocks = <&clkc CLKID_AIU_GLUE>, + <&clkc CLKID_I2S_OUT>, + <&clkc CLKID_AOCLK_GATE>, + <&clkc CLKID_CTS_AMCLK>, + <&clkc CLKID_MIXER_IFACE>, + <&clkc CLKID_IEC958>, + <&clkc CLKID_IEC958_GATE>, + <&clkc CLKID_CTS_MCLK_I958>, + <&clkc CLKID_CTS_I958>; + clock-names = "pclk", + "i2s_pclk", + "i2s_aoclk", + "i2s_mclk", + "i2s_mixer", + "spdif_pclk", + "spdif_aoclk", + "spdif_mclk", + "spdif_mclk_sel"; + resets = <&reset RESET_AIU>; + }; + diff --git a/Documentation/devicetree/bindings/sound/amlogic,g12a-toacodec.yaml b/Documentation/devicetree/bindings/sound/amlogic,g12a-toacodec.yaml new file mode 100644 index 000000000000..f778d3371fde --- /dev/null +++ b/Documentation/devicetree/bindings/sound/amlogic,g12a-toacodec.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/amlogic,g12a-toacodec.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Amlogic G12a Internal DAC Control Glue + +maintainers: + - Jerome Brunet + +properties: + $nodename: + pattern: "^audio-controller@.*" + + "#sound-dai-cells": + const: 1 + + compatible: + oneOf: + - items: + - const: + amlogic,g12a-toacodec + - items: + - enum: + - amlogic,sm1-toacodec + - const: + amlogic,g12a-toacodec + + reg: + maxItems: 1 + + resets: + maxItems: 1 + +required: + - "#sound-dai-cells" + - compatible + - reg + - resets + +examples: + - | + #include + + toacodec: audio-controller@740 { + compatible = "amlogic,g12a-toacodec"; + reg = <0x0 0x740 0x0 0x4>; + #sound-dai-cells = <1>; + resets = <&clkc_audio AUD_RESET_TOACODEC>; + }; diff --git a/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml b/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml new file mode 100644 index 000000000000..fb374c659be1 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/amlogic,gx-sound-card.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Amlogic GX sound card + +maintainers: + - Jerome Brunet + +properties: + compatible: + items: + - const: amlogic,gx-sound-card + + audio-aux-devs: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: list of auxiliary devices + + audio-routing: + $ref: /schemas/types.yaml#/definitions/non-unique-string-array + minItems: 2 + description: |- + A list of the connections between audio components. Each entry is a + pair of strings, the first being the connection's sink, the second + being the connection's source. + + audio-widgets: + $ref: /schemas/types.yaml#/definitions/non-unique-string-array + minItems: 2 + description: |- + A list off component DAPM widget. Each entry is a pair of strings, + the first being the widget type, the second being the widget name + + model: + $ref: /schemas/types.yaml#/definitions/string + description: User specified audio sound card name + +patternProperties: + "^dai-link-[0-9]+$": + type: object + description: |- + dai-link child nodes: + Container for dai-link level properties and the CODEC sub-nodes. + There should be at least one (and probably more) subnode of this type + + properties: + dai-format: + $ref: /schemas/types.yaml#/definitions/string + enum: [ i2s, left-j, dsp_a ] + + mclk-fs: + $ref: /schemas/types.yaml#/definitions/uint32 + description: |- + Multiplication factor between the frame rate and master clock + rate + + sound-dai: + $ref: /schemas/types.yaml#/definitions/phandle + description: phandle of the CPU DAI + + patternProperties: + "^codec-[0-9]+$": + type: object + description: |- + Codecs: + dai-link representing backend links should have at least one subnode. + One subnode for each codec of the dai-link. dai-link representing + frontend links have no codec, therefore have no subnodes + + properties: + sound-dai: + $ref: /schemas/types.yaml#/definitions/phandle + description: phandle of the codec DAI + + required: + - sound-dai + + required: + - sound-dai + +required: + - model + - dai-link-0 + +examples: + - | + sound { + compatible = "amlogic,gx-sound-card"; + model = "GXL-ACME-S905X-FOO"; + audio-aux-devs = <&>; + audio-routing = "I2S ENCODER I2S IN", "I2S FIFO Playback"; + + dai-link-0 { + sound-dai = <&i2s_fifo>; + }; + + dai-link-1 { + sound-dai = <&i2s_encoder>; + dai-format = "i2s"; + mclk-fs = <256>; + + codec-0 { + sound-dai = <&codec0>; + }; + + codec-1 { + sound-dai = <&codec1>; + }; + }; + }; + diff --git a/Documentation/devicetree/bindings/sound/amlogic,t9015.yaml b/Documentation/devicetree/bindings/sound/amlogic,t9015.yaml new file mode 100644 index 000000000000..b7c38c2b5b54 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/amlogic,t9015.yaml @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/amlogic,t9015.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Amlogic T9015 Internal Audio DAC + +maintainers: + - Jerome Brunet + +properties: + $nodename: + pattern: "^audio-controller@.*" + + "#sound-dai-cells": + const: 0 + + compatible: + items: + - const: amlogic,t9015 + + clocks: + items: + - description: Peripheral clock + + clock-names: + items: + - const: pclk + + reg: + maxItems: 1 + + resets: + maxItems: 1 + +required: + - "#sound-dai-cells" + - compatible + - reg + - clocks + - clock-names + - resets + +examples: + - | + #include + #include + + acodec: audio-controller@32000 { + compatible = "amlogic,t9015"; + reg = <0x0 0x32000 0x0 0x14>; + #sound-dai-cells = <0>; + clocks = <&clkc CLKID_AUDIO_CODEC>; + clock-names = "pclk"; + resets = <&reset RESET_AUDIO_CODEC>; + }; + diff --git a/Documentation/devicetree/bindings/sound/brcm,bcm63xx-audio.txt b/Documentation/devicetree/bindings/sound/brcm,bcm63xx-audio.txt new file mode 100644 index 000000000000..007f524b4d15 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/brcm,bcm63xx-audio.txt @@ -0,0 +1,29 @@ +Broadcom DSL/PON BCM63xx Audio I2S controller + +Required properties: +- compatible: Should be "brcm,bcm63xx-i2s". +- #address-cells: 32bit valued, 1 cell. +- #size-cells: 32bit valued, 0 cell. +- reg: Should contain audio registers location and length +- interrupts: Should contain the interrupt for the controller. +- clocks: Must contain an entry for each entry in clock-names. + Please refer to clock-bindings.txt. +- clock-names: One of each entry matching the clocks phandles list: + - "i2sclk" (generated clock) Required. + - "i2sosc" (fixed 200MHz clock) Required. + +(1) : The generated clock is required only when any of TX and RX + works on Master Mode. +(2) : The fixed 200MHz clock is from internal chip and always on + +Example: + + i2s: bcm63xx-i2s { + #address-cells = <1>; + #size-cells = <0>; + compatible = "brcm,bcm63xx-i2s"; + reg = <0xFF802080 0xFF>; + interrupts = ; + clocks = <&i2sclk>, <&osc>; + clock-names = "i2sclk","i2sosc"; + }; diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l51.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l51.yaml new file mode 100644 index 000000000000..efce847a3408 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l51.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/cirrus,cs42l51.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: CS42L51 audio codec DT bindings + +maintainers: + - Olivier Moysan + +properties: + compatible: + const: cirrus,cs42l51 + + reg: + maxItems: 1 + + "#sound-dai-cells": + const: 0 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: MCLK + + reset-gpios: + maxItems: 1 + + VL-supply: + description: phandle to voltage regulator of digital interface section + + VD-supply: + description: phandle to voltage regulator of digital internal section + + VA-supply: + description: phandle to voltage regulator of analog internal section + + VAHP-supply: + description: phandle to voltage regulator of headphone + +required: + - compatible + - reg + - "#sound-dai-cells" + +examples: + - | + #include + i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + + cs42l51@4a { + compatible = "cirrus,cs42l51"; + reg = <0x4a>; + #sound-dai-cells = <0>; + clocks = <&mclk_prov>; + clock-names = "MCLK"; + VL-supply = <®_audio>; + VD-supply = <®_audio>; + VA-supply = <®_audio>; + VAHP-supply = <®_audio>; + reset-gpios = <&gpiog 9 GPIO_ACTIVE_LOW>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/sound/cs42l51.txt b/Documentation/devicetree/bindings/sound/cs42l51.txt deleted file mode 100644 index acbd68ddd2cb..000000000000 --- a/Documentation/devicetree/bindings/sound/cs42l51.txt +++ /dev/null @@ -1,33 +0,0 @@ -CS42L51 audio CODEC - -Required properties: - - - compatible : "cirrus,cs42l51" - - - reg : the I2C address of the device for I2C. - -Optional properties: - - VL-supply, VD-supply, VA-supply, VAHP-supply: power supplies for the device, - as covered in Documentation/devicetree/bindings/regulator/regulator.txt. - - - reset-gpios : GPIO specification for the reset pin. If specified, it will be - deasserted before starting the communication with the codec. - - - clocks : a list of phandles + clock-specifiers, one for each entry in - clock-names - - - clock-names : must contain "MCLK" - -Example: - -cs42l51: cs42l51@4a { - compatible = "cirrus,cs42l51"; - reg = <0x4a>; - clocks = <&mclk_prov>; - clock-names = "MCLK"; - VL-supply = <®_audio>; - VD-supply = <®_audio>; - VA-supply = <®_audio>; - VAHP-supply = <®_audio>; - reset-gpios = <&gpiog 9 GPIO_ACTIVE_LOW>; -}; diff --git a/Documentation/devicetree/bindings/sound/google,cros-ec-codec.txt b/Documentation/devicetree/bindings/sound/google,cros-ec-codec.txt deleted file mode 100644 index 8ca52dcc5572..000000000000 --- a/Documentation/devicetree/bindings/sound/google,cros-ec-codec.txt +++ /dev/null @@ -1,44 +0,0 @@ -Audio codec controlled by ChromeOS EC - -Google's ChromeOS EC codec is a digital mic codec provided by the -Embedded Controller (EC) and is controlled via a host-command interface. - -An EC codec node should only be found as a sub-node of the EC node (see -Documentation/devicetree/bindings/mfd/cros-ec.txt). - -Required properties: -- compatible: Must contain "google,cros-ec-codec" -- #sound-dai-cells: Should be 1. The cell specifies number of DAIs. - -Optional properties: -- reg: Pysical base address and length of shared memory region from EC. - It contains 3 unsigned 32-bit integer. The first 2 integers - combine to become an unsigned 64-bit physical address. The last - one integer is length of the shared memory. -- memory-region: Shared memory region to EC. A "shared-dma-pool". See - ../reserved-memory/reserved-memory.txt for details. - -Example: - -{ - ... - - reserved_mem: reserved_mem { - compatible = "shared-dma-pool"; - reg = <0 0x52800000 0 0x100000>; - no-map; - }; -} - -cros-ec@0 { - compatible = "google,cros-ec-spi"; - - ... - - cros_ec_codec: ec-codec { - compatible = "google,cros-ec-codec"; - #sound-dai-cells = <1>; - reg = <0x0 0x10500000 0x80000>; - memory-region = <&reserved_mem>; - }; -}; diff --git a/Documentation/devicetree/bindings/sound/google,cros-ec-codec.yaml b/Documentation/devicetree/bindings/sound/google,cros-ec-codec.yaml new file mode 100644 index 000000000000..c84e656afb0a --- /dev/null +++ b/Documentation/devicetree/bindings/sound/google,cros-ec-codec.yaml @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/google,cros-ec-codec.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Audio codec controlled by ChromeOS EC + +maintainers: + - Cheng-Yi Chiang + +description: | + Google's ChromeOS EC codec is a digital mic codec provided by the + Embedded Controller (EC) and is controlled via a host-command interface. + An EC codec node should only be found as a sub-node of the EC node (see + Documentation/devicetree/bindings/mfd/cros-ec.txt). + +properties: + compatible: + const: google,cros-ec-codec + + "#sound-dai-cells": + const: 1 + + reg: + items: + - description: | + Physical base address and length of shared memory region from EC. + It contains 3 unsigned 32-bit integer. The first 2 integers + combine to become an unsigned 64-bit physical address. + The last one integer is the length of the shared memory. + + memory-region: + $ref: '/schemas/types.yaml#/definitions/phandle' + description: | + Shared memory region to EC. A "shared-dma-pool". + See ../reserved-memory/reserved-memory.txt for details. + +required: + - compatible + - '#sound-dai-cells' + +additionalProperties: false + +examples: + - | + reserved_mem: reserved-mem@52800000 { + compatible = "shared-dma-pool"; + reg = <0x52800000 0x100000>; + no-map; + }; + spi { + #address-cells = <1>; + #size-cells = <0>; + cros-ec@0 { + compatible = "google,cros-ec-spi"; + #address-cells = <2>; + #size-cells = <1>; + reg = <0>; + cros_ec_codec: ec-codec@10500000 { + compatible = "google,cros-ec-codec"; + #sound-dai-cells = <1>; + reg = <0x0 0x10500000 0x80000>; + memory-region = <&reserved_mem>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/sound/ingenic,aic.yaml b/Documentation/devicetree/bindings/sound/ingenic,aic.yaml new file mode 100644 index 000000000000..44f49bebb267 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/ingenic,aic.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/ingenic,aic.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Ingenic SoCs AC97 / I2S Controller (AIC) DT bindings + +maintainers: + - Paul Cercueil + +properties: + $nodename: + pattern: '^audio-controller@' + + compatible: + oneOf: + - enum: + - ingenic,jz4740-i2s + - ingenic,jz4760-i2s + - ingenic,jz4770-i2s + - ingenic,jz4780-i2s + - items: + - const: ingenic,jz4725b-i2s + - const: ingenic,jz4740-i2s + + '#sound-dai-cells': + const: 0 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: AIC clock + - description: I2S clock + - description: EXT clock + - description: PLL/2 clock + + clock-names: + items: + - const: aic + - const: i2s + - const: ext + - const: pll half + + dmas: + items: + - description: DMA controller phandle and request line for I2S RX + - description: DMA controller phandle and request line for I2S TX + + dma-names: + items: + - const: rx + - const: tx + +additionalProperties: false + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - dmas + - dma-names + - '#sound-dai-cells' + +examples: + - | + #include + aic: audio-controller@10020000 { + compatible = "ingenic,jz4740-i2s"; + reg = <0x10020000 0x38>; + + #sound-dai-cells = <0>; + + interrupt-parent = <&intc>; + interrupts = <18>; + + clocks = <&cgu JZ4740_CLK_AIC>, + <&cgu JZ4740_CLK_I2S>, + <&cgu JZ4740_CLK_EXT>, + <&cgu JZ4740_CLK_PLL_HALF>; + clock-names = "aic", "i2s", "ext", "pll half"; + + dmas = <&dmac 25 0xffffffff>, <&dmac 24 0xffffffff>; + dma-names = "rx", "tx"; + }; diff --git a/Documentation/devicetree/bindings/sound/ingenic,jz4740-i2s.txt b/Documentation/devicetree/bindings/sound/ingenic,jz4740-i2s.txt deleted file mode 100644 index b623d50004fb..000000000000 --- a/Documentation/devicetree/bindings/sound/ingenic,jz4740-i2s.txt +++ /dev/null @@ -1,23 +0,0 @@ -Ingenic JZ4740 I2S controller - -Required properties: -- compatible : "ingenic,jz4740-i2s" or "ingenic,jz4780-i2s" -- reg : I2S registers location and length -- clocks : AIC and I2S PLL clock specifiers. -- clock-names: "aic" and "i2s" -- dmas: DMA controller phandle and DMA request line for I2S Tx and Rx channels -- dma-names: Must be "tx" and "rx" - -Example: - -i2s: i2s@10020000 { - compatible = "ingenic,jz4740-i2s"; - reg = <0x10020000 0x94>; - - clocks = <&cgu JZ4740_CLK_AIC>, <&cgu JZ4740_CLK_I2SPLL>; - clock-names = "aic", "i2s"; - - dmas = <&dma 2>, <&dma 3>; - dma-names = "tx", "rx"; - -}; diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-wm8903.txt b/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-wm8903.txt index b795d282818d..a8f2b0c56c79 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-wm8903.txt +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra-audio-wm8903.txt @@ -18,6 +18,7 @@ Required properties: * Headphone Jack * Int Spk * Mic Jack + * Int Mic - nvidia,i2s-controller : The phandle of the Tegra I2S1 controller - nvidia,audio-codec : The phandle of the WM8903 audio codec diff --git a/Documentation/devicetree/bindings/sound/rockchip,rk3328-codec.txt b/Documentation/devicetree/bindings/sound/rockchip,rk3328-codec.txt index 2469588c7ccb..1ecd75d2032a 100644 --- a/Documentation/devicetree/bindings/sound/rockchip,rk3328-codec.txt +++ b/Documentation/devicetree/bindings/sound/rockchip,rk3328-codec.txt @@ -10,6 +10,11 @@ Required properties: - clock-names: should be "pclk". - spk-depop-time-ms: speak depop time msec. +Optional properties: + +- mute-gpios: GPIO specifier for external line driver control (typically the + dedicated GPIO_MUTE pin) + Example for rk3328 internal codec: codec: codec@ff410000 { @@ -18,6 +23,6 @@ codec: codec@ff410000 { rockchip,grf = <&grf>; clocks = <&cru PCLK_ACODEC>; clock-names = "pclk"; + mute-gpios = <&grf_gpio 0 GPIO_ACTIVE_LOW>; spk-depop-time-ms = 100; - status = "disabled"; }; diff --git a/Documentation/devicetree/bindings/sound/rockchip-i2s.txt b/Documentation/devicetree/bindings/sound/rockchip-i2s.txt deleted file mode 100644 index 54aefab71f2c..000000000000 --- a/Documentation/devicetree/bindings/sound/rockchip-i2s.txt +++ /dev/null @@ -1,49 +0,0 @@ -* Rockchip I2S controller - -The I2S bus (Inter-IC sound bus) is a serial link for digital -audio data transfer between devices in the system. - -Required properties: - -- compatible: should be one of the following: - - "rockchip,rk3066-i2s": for rk3066 - - "rockchip,px30-i2s", "rockchip,rk3066-i2s": for px30 - - "rockchip,rk3036-i2s", "rockchip,rk3066-i2s": for rk3036 - - "rockchip,rk3188-i2s", "rockchip,rk3066-i2s": for rk3188 - - "rockchip,rk3228-i2s", "rockchip,rk3066-i2s": for rk3228 - - "rockchip,rk3288-i2s", "rockchip,rk3066-i2s": for rk3288 - - "rockchip,rk3328-i2s", "rockchip,rk3066-i2s": for rk3328 - - "rockchip,rk3366-i2s", "rockchip,rk3066-i2s": for rk3366 - - "rockchip,rk3368-i2s", "rockchip,rk3066-i2s": for rk3368 - - "rockchip,rk3399-i2s", "rockchip,rk3066-i2s": for rk3399 -- reg: physical base address of the controller and length of memory mapped - region. -- interrupts: should contain the I2S interrupt. -- dmas: DMA specifiers for tx and rx dma. See the DMA client binding, - Documentation/devicetree/bindings/dma/dma.txt -- dma-names: should include "tx" and "rx". -- clocks: a list of phandle + clock-specifer pairs, one for each entry in clock-names. -- clock-names: should contain the following: - - "i2s_hclk": clock for I2S BUS - - "i2s_clk" : clock for I2S controller -- rockchip,playback-channels: max playback channels, if not set, 8 channels default. -- rockchip,capture-channels: max capture channels, if not set, 2 channels default. - -Required properties for controller which support multi channels -playback/capture: - -- rockchip,grf: the phandle of the syscon node for GRF register. - -Example for rk3288 I2S controller: - -i2s@ff890000 { - compatible = "rockchip,rk3288-i2s", "rockchip,rk3066-i2s"; - reg = <0xff890000 0x10000>; - interrupts = ; - dmas = <&pdma1 0>, <&pdma1 1>; - dma-names = "tx", "rx"; - clock-names = "i2s_hclk", "i2s_clk"; - clocks = <&cru HCLK_I2S0>, <&cru SCLK_I2S0>; - rockchip,playback-channels = <8>; - rockchip,capture-channels = <2>; -}; diff --git a/Documentation/devicetree/bindings/sound/rockchip-i2s.yaml b/Documentation/devicetree/bindings/sound/rockchip-i2s.yaml new file mode 100644 index 000000000000..7cd0e278ed85 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/rockchip-i2s.yaml @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/rockchip-i2s.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Rockchip I2S controller + +description: + The I2S bus (Inter-IC sound bus) is a serial link for digital + audio data transfer between devices in the system. + +maintainers: + - Heiko Stuebner + +properties: + compatible: + oneOf: + - const: rockchip,rk3066-i2s + - items: + - enum: + - rockchip,px30-i2s + - rockchip,rk3036-i2s + - rockchip,rk3188-i2s + - rockchip,rk3228-i2s + - rockchip,rk3288-i2s + - rockchip,rk3328-i2s + - rockchip,rk3366-i2s + - rockchip,rk3368-i2s + - rockchip,rk3399-i2s + - const: rockchip,rk3066-i2s + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: clock for I2S controller + - description: clock for I2S BUS + + clock-names: + items: + - const: i2s_clk + - const: i2s_hclk + + dmas: + items: + - description: TX DMA Channel + - description: RX DMA Channel + + dma-names: + items: + - const: tx + - const: rx + + rockchip,capture-channels: + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + default: 2 + description: + Max capture channels, if not set, 2 channels default. + + rockchip,playback-channels: + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + default: 8 + description: + Max playback channels, if not set, 8 channels default. + + rockchip,grf: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The phandle of the syscon node for the GRF register. + Required property for controllers which support multi channel + playback/capture. + + "#sound-dai-cells": + const: 0 + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - dmas + - dma-names + - "#sound-dai-cells" + +additionalProperties: false + +examples: + - | + #include + #include + #include + i2s@ff890000 { + compatible = "rockchip,rk3288-i2s", "rockchip,rk3066-i2s"; + reg = <0xff890000 0x10000>; + interrupts = ; + clocks = <&cru SCLK_I2S0>, <&cru HCLK_I2S0>; + clock-names = "i2s_clk", "i2s_hclk"; + dmas = <&pdma1 0>, <&pdma1 1>; + dma-names = "tx", "rx"; + rockchip,capture-channels = <2>; + rockchip,playback-channels = <8>; + #sound-dai-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/sound/rt5682.txt b/Documentation/devicetree/bindings/sound/rt5682.txt index 30e927a28369..ade1ece8b45f 100644 --- a/Documentation/devicetree/bindings/sound/rt5682.txt +++ b/Documentation/devicetree/bindings/sound/rt5682.txt @@ -32,6 +32,18 @@ Optional properties: The delay time is realtek,btndet-delay value multiple of 8.192 ms. If absent, the default is 16. +- #clock-cells : Should be set to '<1>', wclk and bclk sources provided. +- clock-output-names : Name given for DAI clocks output. + +- clocks : phandle and clock specifier for codec MCLK. +- clock-names : Clock name string for 'clocks' attribute, should be "mclk". + +- realtek,dmic-clk-rate-hz : Set the clock rate (hz) for the requirement of + the particular DMIC. + +- realtek,dmic-delay-ms : Set the delay time (ms) for the requirement of + the particular DMIC. + Pins on the device (for linking into audio routes) for RT5682: * DMIC L1 @@ -53,4 +65,10 @@ rt5682 { realtek,dmic1-clk-pin = <1>; realtek,jd-src = <1>; realtek,btndet-delay = <16>; + + #clock-cells = <1>; + clock-output-names = "rt5682-dai-wclk", "rt5682-dai-bclk"; + + clocks = <&osc>; + clock-names = "mclk"; }; diff --git a/Documentation/devicetree/bindings/sound/st,stm32-i2s.txt b/Documentation/devicetree/bindings/sound/st,stm32-i2s.txt deleted file mode 100644 index cbf24bcd1b8d..000000000000 --- a/Documentation/devicetree/bindings/sound/st,stm32-i2s.txt +++ /dev/null @@ -1,62 +0,0 @@ -STMicroelectronics STM32 SPI/I2S Controller - -The SPI/I2S block supports I2S/PCM protocols when configured on I2S mode. -Only some SPI instances support I2S. - -Required properties: - - compatible: Must be "st,stm32h7-i2s" - - reg: Offset and length of the device's register set. - - interrupts: Must contain the interrupt line id. - - clocks: Must contain phandle and clock specifier pairs for each entry - in clock-names. - - clock-names: Must contain "i2sclk", "pclk", "x8k" and "x11k". - "i2sclk": clock which feeds the internal clock generator - "pclk": clock which feeds the peripheral bus interface - "x8k": I2S parent clock for sampling rates multiple of 8kHz. - "x11k": I2S parent clock for sampling rates multiple of 11.025kHz. - - dmas: DMA specifiers for tx and rx dma. - See Documentation/devicetree/bindings/dma/stm32-dma.txt. - - dma-names: Identifier for each DMA request line. Must be "tx" and "rx". - - pinctrl-names: should contain only value "default" - - pinctrl-0: see Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.yaml - -Optional properties: - - resets: Reference to a reset controller asserting the reset controller - -The device node should contain one 'port' child node with one child 'endpoint' -node, according to the bindings defined in Documentation/devicetree/bindings/ -graph.txt. - -Example: -sound_card { - compatible = "audio-graph-card"; - dais = <&i2s2_port>; -}; - -i2s2: audio-controller@40003800 { - compatible = "st,stm32h7-i2s"; - reg = <0x40003800 0x400>; - interrupts = <36>; - clocks = <&rcc PCLK1>, <&rcc SPI2_CK>, <&rcc PLL1_Q>, <&rcc PLL2_P>; - clock-names = "pclk", "i2sclk", "x8k", "x11k"; - dmas = <&dmamux2 2 39 0x400 0x1>, - <&dmamux2 3 40 0x400 0x1>; - dma-names = "rx", "tx"; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_i2s2>; - - i2s2_port: port@0 { - cpu_endpoint: endpoint { - remote-endpoint = <&codec_endpoint>; - format = "i2s"; - }; - }; -}; - -audio-codec { - codec_port: port@0 { - codec_endpoint: endpoint { - remote-endpoint = <&cpu_endpoint>; - }; - }; -}; diff --git a/Documentation/devicetree/bindings/sound/st,stm32-i2s.yaml b/Documentation/devicetree/bindings/sound/st,stm32-i2s.yaml new file mode 100644 index 000000000000..f32410890589 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/st,stm32-i2s.yaml @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/st,stm32-i2s.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: STMicroelectronics STM32 SPI/I2S Controller + +maintainers: + - Olivier Moysan + +description: + The SPI/I2S block supports I2S/PCM protocols when configured on I2S mode. + Only some SPI instances support I2S. + +properties: + compatible: + enum: + - st,stm32h7-i2s + + "#sound-dai-cells": + const: 0 + + reg: + maxItems: 1 + + clocks: + items: + - description: clock feeding the peripheral bus interface. + - description: clock feeding the internal clock generator. + - description: I2S parent clock for sampling rates multiple of 8kHz. + - description: I2S parent clock for sampling rates multiple of 11.025kHz. + + clock-names: + items: + - const: pclk + - const: i2sclk + - const: x8k + - const: x11k + + interrupts: + maxItems: 1 + + dmas: + items: + - description: audio capture DMA. + - description: audio playback DMA. + + dma-names: + items: + - const: rx + - const: tx + + resets: + maxItems: 1 + +required: + - compatible + - "#sound-dai-cells" + - reg + - clocks + - clock-names + - interrupts + - dmas + - dma-names + +additionalProperties: false + +examples: + - | + #include + #include + i2s2: audio-controller@4000b000 { + compatible = "st,stm32h7-i2s"; + #sound-dai-cells = <0>; + reg = <0x4000b000 0x400>; + clocks = <&rcc SPI2>, <&rcc SPI2_K>, <&rcc PLL3_Q>, <&rcc PLL3_R>; + clock-names = "pclk", "i2sclk", "x8k", "x11k"; + interrupts = ; + dmas = <&dmamux1 39 0x400 0x01>, + <&dmamux1 40 0x400 0x01>; + dma-names = "rx", "tx"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s2_pins_a>; + }; + +... diff --git a/Documentation/devicetree/bindings/sound/st,stm32-spdifrx.txt b/Documentation/devicetree/bindings/sound/st,stm32-spdifrx.txt deleted file mode 100644 index ca9101777c44..000000000000 --- a/Documentation/devicetree/bindings/sound/st,stm32-spdifrx.txt +++ /dev/null @@ -1,56 +0,0 @@ -STMicroelectronics STM32 S/PDIF receiver (SPDIFRX). - -The SPDIFRX peripheral, is designed to receive an S/PDIF flow compliant with -IEC-60958 and IEC-61937. - -Required properties: - - compatible: should be "st,stm32h7-spdifrx" - - reg: cpu DAI IP base address and size - - clocks: must contain an entry for kclk (used as S/PDIF signal reference) - - clock-names: must contain "kclk" - - interrupts: cpu DAI interrupt line - - dmas: DMA specifiers for audio data DMA and iec control flow DMA - See STM32 DMA bindings, Documentation/devicetree/bindings/dma/st,stm32-dma.yaml - - dma-names: two dmas have to be defined, "rx" and "rx-ctrl" - -Optional properties: - - resets: Reference to a reset controller asserting the SPDIFRX - -The device node should contain one 'port' child node with one child 'endpoint' -node, according to the bindings defined in Documentation/devicetree/bindings/ -graph.txt. - -Example: -spdifrx: spdifrx@40004000 { - compatible = "st,stm32h7-spdifrx"; - reg = <0x40004000 0x400>; - clocks = <&rcc SPDIFRX_CK>; - clock-names = "kclk"; - interrupts = <97>; - dmas = <&dmamux1 2 93 0x400 0x0>, - <&dmamux1 3 94 0x400 0x0>; - dma-names = "rx", "rx-ctrl"; - pinctrl-0 = <&spdifrx_pins>; - pinctrl-names = "default"; - - spdifrx_port: port { - cpu_endpoint: endpoint { - remote-endpoint = <&codec_endpoint>; - }; - }; -}; - -spdif_in: spdif-in { - compatible = "linux,spdif-dir"; - - codec_port: port { - codec_endpoint: endpoint { - remote-endpoint = <&cpu_endpoint>; - }; - }; -}; - -soundcard { - compatible = "audio-graph-card"; - dais = <&spdifrx_port>; -}; diff --git a/Documentation/devicetree/bindings/sound/st,stm32-spdifrx.yaml b/Documentation/devicetree/bindings/sound/st,stm32-spdifrx.yaml new file mode 100644 index 000000000000..b7f7dc452231 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/st,stm32-spdifrx.yaml @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/st,stm32-spdifrx.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: STMicroelectronics STM32 S/PDIF receiver (SPDIFRX) + +maintainers: + - Olivier Moysan + +description: | + The SPDIFRX peripheral, is designed to receive an S/PDIF flow compliant with + IEC-60958 and IEC-61937. + +properties: + compatible: + enum: + - st,stm32h7-spdifrx + + "#sound-dai-cells": + const: 0 + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: kclk + + interrupts: + maxItems: 1 + + dmas: + items: + - description: audio data capture DMA + - description: IEC status bits capture DMA + + dma-names: + items: + - const: rx + - const: rx-ctrl + + resets: + maxItems: 1 + +required: + - compatible + - "#sound-dai-cells" + - reg + - clocks + - clock-names + - interrupts + - dmas + - dma-names + +additionalProperties: false + +examples: + - | + #include + #include + spdifrx: spdifrx@40004000 { + compatible = "st,stm32h7-spdifrx"; + #sound-dai-cells = <0>; + reg = <0x40004000 0x400>; + clocks = <&rcc SPDIF_K>; + clock-names = "kclk"; + interrupts = ; + dmas = <&dmamux1 2 93 0x400 0x0>, + <&dmamux1 3 94 0x400 0x0>; + dma-names = "rx", "rx-ctrl"; + pinctrl-0 = <&spdifrx_pins>; + pinctrl-names = "default"; + }; + +... diff --git a/Documentation/devicetree/bindings/sound/tas2562.txt b/Documentation/devicetree/bindings/sound/tas2562.txt index 658e1fb18a99..94796b547184 100644 --- a/Documentation/devicetree/bindings/sound/tas2562.txt +++ b/Documentation/devicetree/bindings/sound/tas2562.txt @@ -8,7 +8,7 @@ real time monitoring of loudspeaker behavior. Required properties: - #address-cells - Should be <1>. - #size-cells - Should be <0>. - - compatible: - Should contain "ti,tas2562". + - compatible: - Should contain "ti,tas2562", "ti,tas2563". - reg: - The i2c address. Should be 0x4c, 0x4d, 0x4e or 0x4f. - ti,imon-slot-no:- TDM TX current sense time slot. diff --git a/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml b/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml new file mode 100644 index 000000000000..ab2268c0ee67 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: (GPL-2.0+ OR BSD-2-Clause) +# Copyright (C) 2019 Texas Instruments Incorporated +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/tlv320adcx140.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments TLV320ADCX140 Quad Channel Analog-to-Digital Converter + +maintainers: + - Dan Murphy + +description: | + The TLV320ADCX140 are multichannel (4-ch analog recording or 8-ch digital + PDM microphones recording), high-performance audio, analog-to-digital + converter (ADC) with analog inputs supporting up to 2V RMS. The TLV320ADCX140 + family supports line and microphone Inputs, and offers a programmable + microphone bias or supply voltage generation. + + Specifications can be found at: + http://www.ti.com/lit/ds/symlink/tlv320adc3140.pdf + http://www.ti.com/lit/ds/symlink/tlv320adc5140.pdf + http://www.ti.com/lit/ds/symlink/tlv320adc6140.pdf + +properties: + compatible: + oneOf: + - const: ti,tlv320adc3140 + - const: ti,tlv320adc5140 + - const: ti,tlv320adc6140 + + reg: + maxItems: 1 + description: | + I2C addresss of the device can be one of these 0x4c, 0x4d, 0x4e or 0x4f + + reset-gpios: + description: | + GPIO used for hardware reset. + + areg-supply: + description: | + Regulator with AVDD at 3.3V. If not defined then the internal regulator + is enabled. + + ti,mic-bias-source: + description: | + Indicates the source for MIC Bias. + 0 - Mic bias is set to VREF + 1 - Mic bias is set to VREF × 1.096 + 6 - Mic bias is set to AVDD + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + - enum: [0, 1, 6] + + ti,vref-source: + description: | + Indicates the source for MIC Bias. + 0 - Set VREF to 2.75V + 1 - Set VREF to 2.5V + 2 - Set VREF to 1.375V + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + - enum: [0, 1, 2] + +required: + - compatible + - reg + +examples: + - | + #include + i2c0 { + #address-cells = <1>; + #size-cells = <0>; + codec: codec@4c { + compatible = "ti,tlv320adc5140"; + reg = <0x4c>; + ti,mic-bias-source = <6>; + reset-gpios = <&gpio0 14 GPIO_ACTIVE_HIGH>; + }; + }; diff --git a/Documentation/i2c/smbus-protocol.rst b/Documentation/i2c/smbus-protocol.rst index c122ed239f7f..c2e29633071e 100644 --- a/Documentation/i2c/smbus-protocol.rst +++ b/Documentation/i2c/smbus-protocol.rst @@ -274,7 +274,7 @@ to know which slave triggered the interrupt. This is implemented the following way in the Linux kernel: * I2C bus drivers which support SMBus alert should call - i2c_setup_smbus_alert() to setup SMBus alert support. + i2c_new_smbus_alert_device() to install SMBus alert support. * I2C drivers for devices which can trigger SMBus alerts should implement the optional alert() callback. diff --git a/Documentation/remoteproc.txt b/Documentation/remoteproc.txt index 03c3d2e568b0..2be1147256e0 100644 --- a/Documentation/remoteproc.txt +++ b/Documentation/remoteproc.txt @@ -230,7 +230,7 @@ in the used rings. Binary Firmware Structure ========================= -At this point remoteproc only supports ELF32 firmware binaries. However, +At this point remoteproc supports ELF32 and ELF64 firmware binaries. However, it is quite expected that other platforms/devices which we'd want to support with this framework will be based on different binary formats. diff --git a/Documentation/sound/alsa-configuration.rst b/Documentation/sound/alsa-configuration.rst index 392875a1b94e..72f97d4b01a7 100644 --- a/Documentation/sound/alsa-configuration.rst +++ b/Documentation/sound/alsa-configuration.rst @@ -2234,6 +2234,19 @@ use_vmalloc buffers. If mmap is used on such architectures, turn off this option, so that the DMA-coherent buffers are allocated and used instead. +delayed_register + The option is needed for devices that have multiple streams + defined in multiple USB interfaces. The driver may invoke + registrations multiple times (once per interface) and this may + lead to the insufficient device enumeration. + This option receives an array of strings, and you can pass + ID:INTERFACE like ``0123abcd:4`` for performing the delayed + registration to the given device. In this example, when a USB + device 0123:abcd is probed, the driver waits the registration + until the USB interface 4 gets probed. + The driver prints a message like "Found post-registration device + assignment: 1234abcd:04" for such a device, so that user can + notice the need. This module supports multiple devices, autoprobe and hotplugging. diff --git a/Documentation/sound/hd-audio/index.rst b/Documentation/sound/hd-audio/index.rst index f8a72ffffe66..6e12de9fc34e 100644 --- a/Documentation/sound/hd-audio/index.rst +++ b/Documentation/sound/hd-audio/index.rst @@ -8,3 +8,4 @@ HD-Audio models controls dp-mst + realtek-pc-beep diff --git a/Documentation/sound/hd-audio/models.rst b/Documentation/sound/hd-audio/models.rst index 11298f0ce44d..0ea967d34583 100644 --- a/Documentation/sound/hd-audio/models.rst +++ b/Documentation/sound/hd-audio/models.rst @@ -216,8 +216,6 @@ alc298-dell-aio ALC298 fixups on Dell AIO machines alc275-dell-xps ALC275 fixups on Dell XPS models -alc256-dell-xps13 - ALC256 fixups on Dell XPS13 lenovo-spk-noise Workaround for speaker noise on Lenovo machines lenovo-hotkey diff --git a/Documentation/sound/hd-audio/realtek-pc-beep.rst b/Documentation/sound/hd-audio/realtek-pc-beep.rst new file mode 100644 index 000000000000..be47c6f76a6e --- /dev/null +++ b/Documentation/sound/hd-audio/realtek-pc-beep.rst @@ -0,0 +1,129 @@ +=============================== +Realtek PC Beep Hidden Register +=============================== + +This file documents the "PC Beep Hidden Register", which is present in certain +Realtek HDA codecs and controls a muxer and pair of passthrough mixers that can +route audio between pins but aren't themselves exposed as HDA widgets. As far +as I can tell, these hidden routes are designed to allow flexible PC Beep output +for codecs that don't have mixer widgets in their output paths. Why it's easier +to hide a mixer behind an undocumented vendor register than to just expose it +as a widget, I have no idea. + +Register Description +==================== + +The register is accessed via processing coefficient 0x36 on NID 20h. Bits not +identified below have no discernible effect on my machine, a Dell XPS 13 9350:: + + MSB LSB + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | |h|S|L| | B |R| | Known bits + +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + |0|0|1|1| 0x7 |0|0x0|1| 0x7 | Reset value + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +1Ah input select (B): 2 bits + When zero, expose the PC Beep line (from the internal beep generator, when + enabled with the Set Beep Generation verb on NID 01h, or else from the + external PCBEEP pin) on the 1Ah pin node. When nonzero, expose the headphone + jack (or possibly Line In on some machines) input instead. If PC Beep is + selected, the 1Ah boost control has no effect. + +Amplify 1Ah loopback, left (L): 1 bit + Amplify the left channel of 1Ah before mixing it into outputs as specified + by h and S bits. Does not affect the level of 1Ah exposed to other widgets. + +Amplify 1Ah loopback, right (R): 1 bit + Amplify the right channel of 1Ah before mixing it into outputs as specified + by h and S bits. Does not affect the level of 1Ah exposed to other widgets. + +Loopback 1Ah to 21h [active low] (h): 1 bit + When zero, mix 1Ah (possibly with amplification, depending on L and R bits) + into 21h (headphone jack on my machine). Mixed signal respects the mute + setting on 21h. + +Loopback 1Ah to 14h (S): 1 bit + When one, mix 1Ah (possibly with amplification, depending on L and R bits) + into 14h (internal speaker on my machine). Mixed signal **ignores** the mute + setting on 14h and is present whenever 14h is configured as an output. + +Path diagrams +============= + +1Ah input selection (DIV is the PC Beep divider set on NID 01h):: + + + | | | + +--DIV--+--!DIV--+ {1Ah boost control} + | | + +--(b == 0)--+--(b != 0)--+ + | + >1Ah (Beep/Headphone Mic/Line In)< + +Loopback of 1Ah to 21h/14h:: + + <1Ah (Beep/Headphone Mic/Line In)> + | + {amplify if L/R} + | + +-----!h-----+-----S-----+ + | | + {21h mute control} | + | | + >21h (Headphone)< >14h (Internal Speaker)< + +Background +========== + +All Realtek HDA codecs have a vendor-defined widget with node ID 20h which +provides access to a bank of registers that control various codec functions. +Registers are read and written via the standard HDA processing coefficient +verbs (Set/Get Coefficient Index, Set/Get Processing Coefficient). The node is +named "Realtek Vendor Registers" in public datasheets' verb listings and, +apart from that, is entirely undocumented. + +This particular register, exposed at coefficient 0x36 and named in commits from +Realtek, is of note: unlike most registers, which seem to control detailed +amplifier parameters not in scope of the HDA specification, it controls audio +routing which could just as easily have been defined using standard HDA mixer +and selector widgets. + +Specifically, it selects between two sources for the input pin widget with Node +ID (NID) 1Ah: the widget's signal can come either from an audio jack (on my +laptop, a Dell XPS 13 9350, it's the headphone jack, but comments in Realtek +commits indicate that it might be a Line In on some machines) or from the PC +Beep line (which is itself multiplexed between the codec's internal beep +generator and external PCBEEP pin, depending on if the beep generator is +enabled via verbs on NID 01h). Additionally, it can mix (with optional +amplification) that signal onto the 21h and/or 14h output pins. + +The register's reset value is 0x3717, corresponding to PC Beep on 1Ah that is +then amplified and mixed into both the headphones and the speakers. Not only +does this violate the HDA specification, which says that "[a vendor defined +beep input pin] connection may be maintained *only* while the Link reset +(**RST#**) is asserted", it means that we cannot ignore the register if we care +about the input that 1Ah would otherwise expose or if the PCBEEP trace is +poorly shielded and picks up chassis noise (both of which are the case on my +machine). + +Unfortunately, there are lots of ways to get this register configuration wrong. +Linux, it seems, has gone through most of them. For one, the register resets +after S3 suspend: judging by existing code, this isn't the case for all vendor +registers, and it's led to some fixes that improve behavior on cold boot but +don't last after suspend. Other fixes have successfully switched the 1Ah input +away from PC Beep but have failed to disable both loopback paths. On my +machine, this means that the headphone input is amplified and looped back to +the headphone output, which uses the exact same pins! As you might expect, this +causes terrible headphone noise, the character of which is controlled by the +1Ah boost control. (If you've seen instructions online to fix XPS 13 headphone +noise by changing "Headphone Mic Boost" in ALSA, now you know why.) + +The information here has been obtained through black-box reverse engineering of +the ALC256 codec's behavior and is not guaranteed to be correct. It likely +also applies for the ALC255, ALC257, ALC235, and ALC236, since those codecs +seem to be close relatives of the ALC256. (They all share one initialization +function.) Additionally, other codecs like the ALC225 and ALC285 also have this +register, judging by existing fixups in ``patch_realtek.c``, but specific +data (e.g. node IDs, bit positions, pin mappings) for those codecs may differ +from what I've described here. diff --git a/Documentation/sound/soc/codec-to-codec.rst b/Documentation/sound/soc/codec-to-codec.rst index 810109d7500d..4eaa9a0c41fc 100644 --- a/Documentation/sound/soc/codec-to-codec.rst +++ b/Documentation/sound/soc/codec-to-codec.rst @@ -104,5 +104,10 @@ Make sure to name your corresponding cpu and codec playback and capture dai names ending with "Playback" and "Capture" respectively as dapm core will link and power those dais based on the name. -Note that in current device tree there is no way to mark a dai_link -as codec to codec. However, it may change in future. +A dai_link in a "simple-audio-card" will automatically be detected as +codec to codec when all DAIs on the link belong to codec components. +The dai_link will be initialized with the subset of stream parameters +(channels, format, sample rate) supported by all DAIs on the link. Since +there is no way to provide these parameters in the device tree, this is +mostly useful for communication with simple fixed-function codecs, such +as a Bluetooth controller or cellular modem. diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index ebd383fba939..efbbe570aa9b 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -1574,8 +1574,8 @@ This ioctl would set vcpu's xcr to the value userspace specified. }; #define KVM_CPUID_FLAG_SIGNIFCANT_INDEX BIT(0) - #define KVM_CPUID_FLAG_STATEFUL_FUNC BIT(1) - #define KVM_CPUID_FLAG_STATE_READ_NEXT BIT(2) + #define KVM_CPUID_FLAG_STATEFUL_FUNC BIT(1) /* deprecated */ + #define KVM_CPUID_FLAG_STATE_READ_NEXT BIT(2) /* deprecated */ struct kvm_cpuid_entry2 { __u32 function; @@ -1626,13 +1626,6 @@ emulate them efficiently. The fields in each entry are defined as follows: KVM_CPUID_FLAG_SIGNIFCANT_INDEX: if the index field is valid - KVM_CPUID_FLAG_STATEFUL_FUNC: - if cpuid for this function returns different values for successive - invocations; there will be several entries with the same function, - all with this flag set - KVM_CPUID_FLAG_STATE_READ_NEXT: - for KVM_CPUID_FLAG_STATEFUL_FUNC entries, set if this entry is - the first entry to be read by a cpu eax, ebx, ecx, edx: the values returned by the cpuid instruction for @@ -2117,7 +2110,8 @@ Errors: ====== ============================================================  ENOENT   no such register -  EINVAL   invalid register ID, or no such register +  EINVAL   invalid register ID, or no such register or used with VMs in + protected virtualization mode on s390  EPERM    (arm64) register access not allowed before vcpu finalization ====== ============================================================ @@ -2552,7 +2546,8 @@ Errors include: ======== ============================================================  ENOENT   no such register -  EINVAL   invalid register ID, or no such register +  EINVAL   invalid register ID, or no such register or used with VMs in + protected virtualization mode on s390  EPERM    (arm64) register access not allowed before vcpu finalization ======== ============================================================ @@ -3347,8 +3342,8 @@ The member 'flags' is used for passing flags from userspace. :: #define KVM_CPUID_FLAG_SIGNIFCANT_INDEX BIT(0) - #define KVM_CPUID_FLAG_STATEFUL_FUNC BIT(1) - #define KVM_CPUID_FLAG_STATE_READ_NEXT BIT(2) + #define KVM_CPUID_FLAG_STATEFUL_FUNC BIT(1) /* deprecated */ + #define KVM_CPUID_FLAG_STATE_READ_NEXT BIT(2) /* deprecated */ struct kvm_cpuid_entry2 { __u32 function; @@ -3394,13 +3389,6 @@ The fields in each entry are defined as follows: KVM_CPUID_FLAG_SIGNIFCANT_INDEX: if the index field is valid - KVM_CPUID_FLAG_STATEFUL_FUNC: - if cpuid for this function returns different values for successive - invocations; there will be several entries with the same function, - all with this flag set - KVM_CPUID_FLAG_STATE_READ_NEXT: - for KVM_CPUID_FLAG_STATEFUL_FUNC entries, set if this entry is - the first entry to be read by a cpu eax, ebx, ecx, edx: @@ -4649,6 +4637,60 @@ the clear cpu reset definition in the POP. However, the cpu is not put into ESA mode. This reset is a superset of the initial reset. +4.125 KVM_S390_PV_COMMAND +------------------------- + +:Capability: KVM_CAP_S390_PROTECTED +:Architectures: s390 +:Type: vm ioctl +:Parameters: struct kvm_pv_cmd +:Returns: 0 on success, < 0 on error + +:: + + struct kvm_pv_cmd { + __u32 cmd; /* Command to be executed */ + __u16 rc; /* Ultravisor return code */ + __u16 rrc; /* Ultravisor return reason code */ + __u64 data; /* Data or address */ + __u32 flags; /* flags for future extensions. Must be 0 for now */ + __u32 reserved[3]; + }; + +cmd values: + +KVM_PV_ENABLE + Allocate memory and register the VM with the Ultravisor, thereby + donating memory to the Ultravisor that will become inaccessible to + KVM. All existing CPUs are converted to protected ones. After this + command has succeeded, any CPU added via hotplug will become + protected during its creation as well. + + Errors: + + ===== ============================= + EINTR an unmasked signal is pending + ===== ============================= + +KVM_PV_DISABLE + + Deregister the VM from the Ultravisor and reclaim the memory that + had been donated to the Ultravisor, making it usable by the kernel + again. All registered VCPUs are converted back to non-protected + ones. + +KVM_PV_VM_SET_SEC_PARMS + Pass the image header from VM memory to the Ultravisor in + preparation of image unpacking and verification. + +KVM_PV_VM_UNPACK + Unpack (protect and decrypt) a page of the encrypted boot image. + +KVM_PV_VM_VERIFY + Verify the integrity of the unpacked image. Only if this succeeds, + KVM is allowed to start protected VCPUs. + + 5. The kvm_run structure ======================== @@ -5707,8 +5749,13 @@ and injected exceptions. :Architectures: x86, arm, arm64, mips :Parameters: args[0] whether feature should be enabled or not -With this capability enabled, KVM_GET_DIRTY_LOG will not automatically -clear and write-protect all pages that are returned as dirty. +Valid flags are:: + + #define KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE (1 << 0) + #define KVM_DIRTY_LOG_INITIALLY_SET (1 << 1) + +With KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE is set, KVM_GET_DIRTY_LOG will not +automatically clear and write-protect all pages that are returned as dirty. Rather, userspace will have to do this operation separately using KVM_CLEAR_DIRTY_LOG. @@ -5719,18 +5766,42 @@ than requiring to sync a full memslot; this ensures that KVM does not take spinlocks for an extended period of time. Second, in some cases a large amount of time can pass between a call to KVM_GET_DIRTY_LOG and userspace actually using the data in the page. Pages can be modified -during this time, which is inefficint for both the guest and userspace: +during this time, which is inefficient for both the guest and userspace: the guest will incur a higher penalty due to write protection faults, while userspace can see false reports of dirty pages. Manual reprotection helps reducing this time, improving guest performance and reducing the number of dirty log false positives. +With KVM_DIRTY_LOG_INITIALLY_SET set, all the bits of the dirty bitmap +will be initialized to 1 when created. This also improves performance because +dirty logging can be enabled gradually in small chunks on the first call +to KVM_CLEAR_DIRTY_LOG. KVM_DIRTY_LOG_INITIALLY_SET depends on +KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE (it is also only available on +x86 for now). + KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 was previously available under the name KVM_CAP_MANUAL_DIRTY_LOG_PROTECT, but the implementation had bugs that make it hard or impossible to use it correctly. The availability of KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 signals that those bugs are fixed. Userspace should not try to use KVM_CAP_MANUAL_DIRTY_LOG_PROTECT. +7.19 KVM_CAP_PPC_SECURE_GUEST +------------------------------ + +:Architectures: ppc + +This capability indicates that KVM is running on a host that has +ultravisor firmware and thus can support a secure guest. On such a +system, a guest can ask the ultravisor to make it a secure guest, +one whose memory is inaccessible to the host except for pages which +are explicitly requested to be shared with the host. The ultravisor +notifies KVM when a guest requests to become a secure guest, and KVM +has the opportunity to veto the transition. + +If present, this capability can be enabled for a VM, meaning that KVM +will allow the transition to secure guest mode. Otherwise KVM will +veto the transition. + 8. Other capabilities. ====================== @@ -6027,3 +6098,14 @@ Architectures: s390 This capability indicates that the KVM_S390_NORMAL_RESET and KVM_S390_CLEAR_RESET ioctls are available. + +8.23 KVM_CAP_S390_PROTECTED + +Architecture: s390 + + +This capability indicates that the Ultravisor has been initialized and +KVM can therefore start protected VMs. +This capability governs the KVM_S390_PV_COMMAND ioctl and the +KVM_MP_STATE_LOAD MP_STATE. KVM_SET_MP_STATE can fail for protected +guests when the state change is invalid. diff --git a/Documentation/virt/kvm/arm/hyp-abi.rst b/Documentation/virt/kvm/arm/hyp-abi.rst index d1fc27d848e9..d9eba93aa364 100644 --- a/Documentation/virt/kvm/arm/hyp-abi.rst +++ b/Documentation/virt/kvm/arm/hyp-abi.rst @@ -11,6 +11,11 @@ hypervisor when running as a guest (under Xen, KVM or any other hypervisor), or any hypervisor-specific interaction when the kernel is used as a host. +Note: KVM/arm has been removed from the kernel. The API described +here is still valid though, as it allows the kernel to kexec when +booted at HYP. It can also be used by a hypervisor other than KVM +if necessary. + On arm and arm64 (without VHE), the kernel doesn't run in hypervisor mode, but still needs to interact with it, allowing a built-in hypervisor to be either installed or torn down. diff --git a/Documentation/virt/kvm/devices/s390_flic.rst b/Documentation/virt/kvm/devices/s390_flic.rst index 954190da7d04..ea96559ba501 100644 --- a/Documentation/virt/kvm/devices/s390_flic.rst +++ b/Documentation/virt/kvm/devices/s390_flic.rst @@ -108,16 +108,9 @@ Groups: mask or unmask the adapter, as specified in mask KVM_S390_IO_ADAPTER_MAP - perform a gmap translation for the guest address provided in addr, - pin a userspace page for the translated address and add it to the - list of mappings - - .. note:: A new mapping will be created unconditionally; therefore, - the calling code should avoid making duplicate mappings. - + This is now a no-op. The mapping is purely done by the irq route. KVM_S390_IO_ADAPTER_UNMAP - release a userspace page for the translated address specified in addr - from the list of mappings + This is now a no-op. The mapping is purely done by the irq route. KVM_DEV_FLIC_AISM modify the adapter-interruption-suppression mode for a given isc if the diff --git a/Documentation/virt/kvm/index.rst b/Documentation/virt/kvm/index.rst index 774deaebf7fa..dcc252634cf9 100644 --- a/Documentation/virt/kvm/index.rst +++ b/Documentation/virt/kvm/index.rst @@ -18,6 +18,8 @@ KVM nested-vmx ppc-pv s390-diag + s390-pv + s390-pv-boot timekeeping vcpu-requests diff --git a/Documentation/virt/kvm/locking.rst b/Documentation/virt/kvm/locking.rst index c02291beac3f..b21a34c34a21 100644 --- a/Documentation/virt/kvm/locking.rst +++ b/Documentation/virt/kvm/locking.rst @@ -96,19 +96,18 @@ will happen: We dirty-log for gfn1, that means gfn2 is lost in dirty-bitmap. For direct sp, we can easily avoid it since the spte of direct sp is fixed -to gfn. For indirect sp, before we do cmpxchg, we call gfn_to_pfn_atomic() -to pin gfn to pfn, because after gfn_to_pfn_atomic(): +to gfn. For indirect sp, we disabled fast page fault for simplicity. + +A solution for indirect sp could be to pin the gfn, for example via +kvm_vcpu_gfn_to_pfn_atomic, before the cmpxchg. After the pinning: - We have held the refcount of pfn that means the pfn can not be freed and be reused for another gfn. -- The pfn is writable that means it can not be shared between different gfns +- The pfn is writable and therefore it cannot be shared between different gfns by KSM. Then, we can ensure the dirty bitmaps is correctly set for a gfn. -Currently, to simplify the whole things, we disable fast page fault for -indirect shadow page. - 2) Dirty bit tracking In the origin code, the spte can be fast updated (non-atomically) if the diff --git a/Documentation/virt/kvm/s390-pv-boot.rst b/Documentation/virt/kvm/s390-pv-boot.rst new file mode 100644 index 000000000000..8b8fa0390409 --- /dev/null +++ b/Documentation/virt/kvm/s390-pv-boot.rst @@ -0,0 +1,84 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====================================== +s390 (IBM Z) Boot/IPL of Protected VMs +====================================== + +Summary +------- +The memory of Protected Virtual Machines (PVMs) is not accessible to +I/O or the hypervisor. In those cases where the hypervisor needs to +access the memory of a PVM, that memory must be made accessible. +Memory made accessible to the hypervisor will be encrypted. See +:doc:`s390-pv` for details." + +On IPL (boot) a small plaintext bootloader is started, which provides +information about the encrypted components and necessary metadata to +KVM to decrypt the protected virtual machine. + +Based on this data, KVM will make the protected virtual machine known +to the Ultravisor (UV) and instruct it to secure the memory of the +PVM, decrypt the components and verify the data and address list +hashes, to ensure integrity. Afterwards KVM can run the PVM via the +SIE instruction which the UV will intercept and execute on KVM's +behalf. + +As the guest image is just like an opaque kernel image that does the +switch into PV mode itself, the user can load encrypted guest +executables and data via every available method (network, dasd, scsi, +direct kernel, ...) without the need to change the boot process. + + +Diag308 +------- +This diagnose instruction is the basic mechanism to handle IPL and +related operations for virtual machines. The VM can set and retrieve +IPL information blocks, that specify the IPL method/devices and +request VM memory and subsystem resets, as well as IPLs. + +For PVMs this concept has been extended with new subcodes: + +Subcode 8: Set an IPL Information Block of type 5 (information block +for PVMs) +Subcode 9: Store the saved block in guest memory +Subcode 10: Move into Protected Virtualization mode + +The new PV load-device-specific-parameters field specifies all data +that is necessary to move into PV mode. + +* PV Header origin +* PV Header length +* List of Components composed of + * AES-XTS Tweak prefix + * Origin + * Size + +The PV header contains the keys and hashes, which the UV will use to +decrypt and verify the PV, as well as control flags and a start PSW. + +The components are for instance an encrypted kernel, kernel parameters +and initrd. The components are decrypted by the UV. + +After the initial import of the encrypted data, all defined pages will +contain the guest content. All non-specified pages will start out as +zero pages on first access. + + +When running in protected virtualization mode, some subcodes will result in +exceptions or return error codes. + +Subcodes 4 and 7, which specify operations that do not clear the guest +memory, will result in specification exceptions. This is because the +UV will clear all memory when a secure VM is removed, and therefore +non-clearing IPL subcodes are not allowed. + +Subcodes 8, 9, 10 will result in specification exceptions. +Re-IPL into a protected mode is only possible via a detour into non +protected mode. + +Keys +---- +Every CEC will have a unique public key to enable tooling to build +encrypted images. +See `s390-tools `_ +for the tooling. diff --git a/Documentation/virt/kvm/s390-pv.rst b/Documentation/virt/kvm/s390-pv.rst new file mode 100644 index 000000000000..774a8c606091 --- /dev/null +++ b/Documentation/virt/kvm/s390-pv.rst @@ -0,0 +1,116 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================================= +s390 (IBM Z) Ultravisor and Protected VMs +========================================= + +Summary +------- +Protected virtual machines (PVM) are KVM VMs that do not allow KVM to +access VM state like guest memory or guest registers. Instead, the +PVMs are mostly managed by a new entity called Ultravisor (UV). The UV +provides an API that can be used by PVMs and KVM to request management +actions. + +Each guest starts in non-protected mode and then may make a request to +transition into protected mode. On transition, KVM registers the guest +and its VCPUs with the Ultravisor and prepares everything for running +it. + +The Ultravisor will secure and decrypt the guest's boot memory +(i.e. kernel/initrd). It will safeguard state changes like VCPU +starts/stops and injected interrupts while the guest is running. + +As access to the guest's state, such as the SIE state description, is +normally needed to be able to run a VM, some changes have been made in +the behavior of the SIE instruction. A new format 4 state description +has been introduced, where some fields have different meanings for a +PVM. SIE exits are minimized as much as possible to improve speed and +reduce exposed guest state. + + +Interrupt injection +------------------- +Interrupt injection is safeguarded by the Ultravisor. As KVM doesn't +have access to the VCPUs' lowcores, injection is handled via the +format 4 state description. + +Machine check, external, IO and restart interruptions each can be +injected on SIE entry via a bit in the interrupt injection control +field (offset 0x54). If the guest cpu is not enabled for the interrupt +at the time of injection, a validity interception is recognized. The +format 4 state description contains fields in the interception data +block where data associated with the interrupt can be transported. + +Program and Service Call exceptions have another layer of +safeguarding; they can only be injected for instructions that have +been intercepted into KVM. The exceptions need to be a valid outcome +of an instruction emulation by KVM, e.g. we can never inject a +addressing exception as they are reported by SIE since KVM has no +access to the guest memory. + + +Mask notification interceptions +------------------------------- +KVM cannot intercept lctl(g) and lpsw(e) anymore in order to be +notified when a PVM enables a certain class of interrupt. As a +replacement, two new interception codes have been introduced: One +indicating that the contents of CRs 0, 6, or 14 have been changed, +indicating different interruption subclasses; and one indicating that +PSW bit 13 has been changed, indicating that a machine check +intervention was requested and those are now enabled. + +Instruction emulation +--------------------- +With the format 4 state description for PVMs, the SIE instruction already +interprets more instructions than it does with format 2. It is not able +to interpret every instruction, but needs to hand some tasks to KVM; +therefore, the SIE and the ultravisor safeguard emulation inputs and outputs. + +The control structures associated with SIE provide the Secure +Instruction Data Area (SIDA), the Interception Parameters (IP) and the +Secure Interception General Register Save Area. Guest GRs and most of +the instruction data, such as I/O data structures, are filtered. +Instruction data is copied to and from the SIDA when needed. Guest +GRs are put into / retrieved from the Secure Interception General +Register Save Area. + +Only GR values needed to emulate an instruction will be copied into this +save area and the real register numbers will be hidden. + +The Interception Parameters state description field still contains the +the bytes of the instruction text, but with pre-set register values +instead of the actual ones. I.e. each instruction always uses the same +instruction text, in order not to leak guest instruction text. +This also implies that the register content that a guest had in r +may be in r from the hypervisor's point of view. + +The Secure Instruction Data Area contains instruction storage +data. Instruction data, i.e. data being referenced by an instruction +like the SCCB for sclp, is moved via the SIDA. When an instruction is +intercepted, the SIE will only allow data and program interrupts for +this instruction to be moved to the guest via the two data areas +discussed before. Other data is either ignored or results in validity +interceptions. + + +Instruction emulation interceptions +----------------------------------- +There are two types of SIE secure instruction intercepts: the normal +and the notification type. Normal secure instruction intercepts will +make the guest pending for instruction completion of the intercepted +instruction type, i.e. on SIE entry it is attempted to complete +emulation of the instruction with the data provided by KVM. That might +be a program exception or instruction completion. + +The notification type intercepts inform KVM about guest environment +changes due to guest instruction interpretation. Such an interception +is recognized, for example, for the store prefix instruction to provide +the new lowcore location. On SIE reentry, any KVM data in the data areas +is ignored and execution continues as if the guest instruction had +completed. For that reason KVM is not allowed to inject a program +interrupt. + +Links +----- +`KVM Forum 2019 presentation `_ diff --git a/MAINTAINERS b/MAINTAINERS index 7b2eb8a78d25..3ec94f0a6b30 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1945,7 +1945,7 @@ F: Documentation/devicetree/bindings/i2c/i2c-lpc2k.txt F: arch/arm/boot/dts/lpc43* F: drivers/i2c/busses/i2c-lpc2k.c F: drivers/memory/pl172.c -F: drivers/mtd/spi-nor/nxp-spifi.c +F: drivers/mtd/spi-nor/controllers/nxp-spifi.c F: drivers/rtc/rtc-lpc24xx.c N: lpc18xx @@ -2743,8 +2743,8 @@ L: linux-aspeed@lists.ozlabs.org (moderated for non-subscribers) L: openbmc@lists.ozlabs.org (moderated for non-subscribers) L: linux-gpio@vger.kernel.org S: Maintained -F: drivers/pinctrl/aspeed/ F: Documentation/devicetree/bindings/pinctrl/aspeed,* +F: drivers/pinctrl/aspeed/ ASPEED SCU INTERRUPT CONTROLLER DRIVER M: Eddie James @@ -3470,7 +3470,7 @@ L: linux-i2c@vger.kernel.org L: bcm-kernel-feedback-list@broadcom.com S: Supported F: drivers/i2c/busses/i2c-brcmstb.c -F: Documentation/devicetree/bindings/i2c/i2c-brcmstb.txt +F: Documentation/devicetree/bindings/i2c/brcm,brcmstb-i2c.yaml BROADCOM BRCMSTB USB2 and USB3 PHY DRIVER M: Al Cooper @@ -4055,8 +4055,8 @@ F: Documentation/devicetree/bindings/sound/google,cros-ec-codec.yaml F: sound/soc/codecs/cros_ec_codec.* CIRRUS LOGIC AUDIO CODEC DRIVERS -M: Brian Austin -M: Paul Handrigan +M: James Schulman +M: David Rhodes L: alsa-devel@alsa-project.org (moderated for non-subscribers) S: Maintained F: sound/soc/codecs/cs* @@ -4958,6 +4958,7 @@ F: drivers/leds/leds-da90??.c F: drivers/mfd/da903x.c F: drivers/mfd/da90??-*.c F: drivers/mfd/da91??-*.c +F: drivers/pinctrl/pinctrl-da90??.c F: drivers/power/supply/da9052-battery.c F: drivers/power/supply/da91??-*.c F: drivers/regulator/da903x.c @@ -7850,6 +7851,10 @@ F: Documentation/ABI/testing/debugfs-hyperv HYPERBUS SUPPORT M: Vignesh Raghavendra +L: linux-mtd@lists.infradead.org +Q: http://patchwork.ozlabs.org/project/linux-mtd/list/ +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git cfi/next +C: irc://irc.oftc.net/mtd S: Supported F: drivers/mtd/hyperbus/ F: include/linux/mtd/hyperbus.h @@ -9244,7 +9249,7 @@ F: virt/kvm/* F: tools/kvm/ F: tools/testing/selftests/kvm/ -KERNEL VIRTUAL MACHINE FOR ARM/ARM64 (KVM/arm, KVM/arm64) +KERNEL VIRTUAL MACHINE FOR ARM64 (KVM/arm64) M: Marc Zyngier R: James Morse R: Julien Thierry @@ -9253,9 +9258,6 @@ L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) L: kvmarm@lists.cs.columbia.edu T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git S: Maintained -F: arch/arm/include/uapi/asm/kvm* -F: arch/arm/include/asm/kvm* -F: arch/arm/kvm/ F: arch/arm64/include/uapi/asm/kvm* F: arch/arm64/include/asm/kvm* F: arch/arm64/kvm/ @@ -9290,6 +9292,7 @@ L: kvm@vger.kernel.org W: http://www.ibm.com/developerworks/linux/linux390/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git S: Supported +F: Documentation/virt/kvm/s390* F: arch/s390/include/uapi/asm/kvm* F: arch/s390/include/asm/gmap.h F: arch/s390/include/asm/kvm* @@ -11553,6 +11556,7 @@ L: linux-mtd@lists.infradead.org W: http://www.linux-mtd.infradead.org/ Q: http://patchwork.ozlabs.org/project/linux-mtd/list/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git nand/next +C: irc://irc.oftc.net/mtd S: Maintained F: drivers/mtd/nand/ F: include/linux/mtd/*nand*.h @@ -13275,21 +13279,13 @@ K: \b(clone_args|kernel_clone_args)\b PIN CONTROL SUBSYSTEM M: Linus Walleij L: linux-gpio@vger.kernel.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git F: Documentation/devicetree/bindings/pinctrl/ F: Documentation/driver-api/pinctl.rst F: drivers/pinctrl/ F: include/linux/pinctrl/ -PIN CONTROLLER - MICROCHIP AT91 -M: Ludovic Desroches -L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) -L: linux-gpio@vger.kernel.org -S: Supported -F: drivers/pinctrl/pinctrl-at91* -F: drivers/gpio/gpio-sama5d2-piobu.c - PIN CONTROLLER - FREESCALE M: Dong Aisheng M: Fabio Estevam @@ -13298,14 +13294,14 @@ M: Stefan Agner R: Pengutronix Kernel Team L: linux-gpio@vger.kernel.org S: Maintained -F: drivers/pinctrl/freescale/ F: Documentation/devicetree/bindings/pinctrl/fsl,* +F: drivers/pinctrl/freescale/ PIN CONTROLLER - INTEL M: Mika Westerberg M: Andy Shevchenko -T: git git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git F: drivers/pinctrl/intel/ PIN CONTROLLER - MEDIATEK @@ -13316,18 +13312,26 @@ F: Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt F: Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt F: drivers/pinctrl/mediatek/ +PIN CONTROLLER - MICROCHIP AT91 +M: Ludovic Desroches +L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) +L: linux-gpio@vger.kernel.org +S: Supported +F: drivers/gpio/gpio-sama5d2-piobu.c +F: drivers/pinctrl/pinctrl-at91* + PIN CONTROLLER - QUALCOMM M: Bjorn Andersson -S: Maintained L: linux-arm-msm@vger.kernel.org +S: Maintained F: Documentation/devicetree/bindings/pinctrl/qcom,*.txt F: drivers/pinctrl/qcom/ PIN CONTROLLER - RENESAS M: Geert Uytterhoeven L: linux-renesas-soc@vger.kernel.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git sh-pfc S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git sh-pfc F: drivers/pinctrl/pinctrl-rz* F: drivers/pinctrl/sh-pfc/ @@ -13337,12 +13341,12 @@ M: Krzysztof Kozlowski M: Sylwester Nawrocki L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) L: linux-samsung-soc@vger.kernel.org (moderated for non-subscribers) -Q: https://patchwork.kernel.org/project/linux-samsung-soc/list/ -T: git git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git +Q: https://patchwork.kernel.org/project/linux-samsung-soc/list/ +F: Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt F: drivers/pinctrl/samsung/ F: include/dt-bindings/pinctrl/samsung.h -F: Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt PIN CONTROLLER - SINGLE M: Tony Lindgren @@ -13355,8 +13359,8 @@ F: drivers/pinctrl/pinctrl-single.c PIN CONTROLLER - ST SPEAR M: Viresh Kumar L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) -W: http://www.st.com/spear S: Maintained +W: http://www.st.com/spear F: drivers/pinctrl/spear/ PISTACHIO SOC SUPPORT @@ -15757,6 +15761,17 @@ F: sound/soc/ F: include/dt-bindings/sound/ F: include/sound/soc* +SOUND - SOUND OPEN FIRMWARE (SOF) DRIVERS +M: Pierre-Louis Bossart +M: Liam Girdwood +M: Ranjani Sridharan +M: Kai Vehmanen +M: Daniel Baluta +L: sound-open-firmware@alsa-project.org (moderated for non-subscribers) +W: https://github.com/thesofproject/linux/ +S: Supported +F: sound/soc/sof/ + SOUNDWIRE SUBSYSTEM M: Vinod Koul M: Sanyog Kale @@ -15831,6 +15846,7 @@ L: linux-mtd@lists.infradead.org W: http://www.linux-mtd.infradead.org/ Q: http://patchwork.ozlabs.org/project/linux-mtd/list/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git spi-nor/next +C: irc://irc.oftc.net/mtd S: Maintained F: drivers/mtd/spi-nor/ F: include/linux/mtd/spi-nor.h diff --git a/arch/alpha/include/asm/Kbuild b/arch/alpha/include/asm/Kbuild index 89e87bbc987f..42911c8340c7 100644 --- a/arch/alpha/include/asm/Kbuild +++ b/arch/alpha/include/asm/Kbuild @@ -1,17 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += syscall_table.h -generic-y += compat.h -generic-y += exec.h generic-y += export.h -generic-y += fb.h -generic-y += irq_work.h generic-y += kvm_para.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += preempt.h -generic-y += sections.h -generic-y += trace_clock.h -generic-y += current.h -generic-y += kprobes.h diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c index 741e61ef9d3f..c2d7b6d7bac7 100644 --- a/arch/alpha/mm/fault.c +++ b/arch/alpha/mm/fault.c @@ -89,7 +89,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr, const struct exception_table_entry *fixup; int si_code = SEGV_MAPERR; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; /* As of EV6, a load into $31/$f31 is a prefetch, and never faults (or is suppressed by the PALcode). Support that for older CPUs @@ -150,7 +150,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr, the fault. */ fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -169,7 +169,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr, else current->min_flt++; if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; + flags |= FAULT_FLAG_TRIED; /* No need to up_read(&mm->mmap_sem) as we would * have already released it in __lock_page_or_retry diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild index 1b505694691e..81f4edec0c2a 100644 --- a/arch/arc/include/asm/Kbuild +++ b/arch/arc/include/asm/Kbuild @@ -1,28 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -generic-y += bugs.h -generic-y += compat.h -generic-y += device.h -generic-y += div64.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h generic-y += extable.h -generic-y += ftrace.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h generic-y += kvm_para.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h generic-y += parport.h -generic-y += percpu.h -generic-y += preempt.h -generic-y += topology.h -generic-y += trace_clock.h generic-y += user.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index fb86bc3e9b35..92b339c7adba 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -100,7 +100,7 @@ void do_page_fault(unsigned long address, struct pt_regs *regs) (regs->ecr_cause == ECR_C_PROTV_INST_FETCH)) exec = 1; - flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + flags = FAULT_FLAG_DEFAULT; if (user_mode(regs)) flags |= FAULT_FLAG_USER; if (write) @@ -133,29 +133,20 @@ void do_page_fault(unsigned long address, struct pt_regs *regs) fault = handle_mm_fault(vma, address, flags); - /* - * Fault retry nuances - */ - if (unlikely(fault & VM_FAULT_RETRY)) { + /* Quick path to respond to signals */ + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + goto no_context; + return; + } - /* - * If fault needs to be retried, handle any pending signals - * first (by returning to user mode). - * mmap_sem already relinquished by core mm for RETRY case - */ - if (fatal_signal_pending(current)) { - if (!user_mode(regs)) - goto no_context; - return; - } - /* - * retry state machine - */ - if (flags & FAULT_FLAG_ALLOW_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; - flags |= FAULT_FLAG_TRIED; - goto retry; - } + /* + * Fault retry nuances, mmap_sem already relinquished by core mm + */ + if (unlikely((fault & VM_FAULT_RETRY) && + (flags & FAULT_FLAG_ALLOW_RETRY))) { + flags |= FAULT_FLAG_TRIED; + goto retry; } bad_area: diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 03bbfc312fe7..66a04f6f4775 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -2090,5 +2090,3 @@ source "drivers/firmware/Kconfig" if CRYPTO source "arch/arm/crypto/Kconfig" endif - -source "arch/arm/kvm/Kconfig" diff --git a/arch/arm/Makefile b/arch/arm/Makefile index 1fc32b611f8a..e1d13d779e08 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -278,7 +278,6 @@ core-$(CONFIG_FPE_NWFPE) += arch/arm/nwfpe/ core-$(CONFIG_FPE_FASTFPE) += $(patsubst $(srctree)/%,%,$(wildcard $(srctree)/arch/arm/fastfpe/)) core-$(CONFIG_VFP) += arch/arm/vfp/ core-$(CONFIG_XEN) += arch/arm/xen/ -core-$(CONFIG_KVM_ARM_HOST) += arch/arm/kvm/ core-$(CONFIG_VDSO) += arch/arm/vdso/ # If we have a machine-specific directory, then include it in the build. diff --git a/arch/arm/configs/axm55xx_defconfig b/arch/arm/configs/axm55xx_defconfig index 6ea7dafa4c9e..46075216ee6d 100644 --- a/arch/arm/configs/axm55xx_defconfig +++ b/arch/arm/configs/axm55xx_defconfig @@ -236,5 +236,3 @@ CONFIG_CRYPTO_GCM=y CONFIG_CRYPTO_XCBC=y CONFIG_CRYPTO_SHA256=y # CONFIG_CRYPTO_ANSI_CPRNG is not set -CONFIG_VIRTUALIZATION=y -CONFIG_KVM=y diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild index fa579b23b4df..383635b68763 100644 --- a/arch/arm/include/asm/Kbuild +++ b/arch/arm/include/asm/Kbuild @@ -1,22 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -generic-y += compat.h -generic-y += current.h generic-y += early_ioremap.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += extable.h generic-y += flat.h -generic-y += irq_regs.h -generic-y += kdebug.h -generic-y += local.h generic-y += local64.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h generic-y += parport.h -generic-y += preempt.h generic-y += seccomp.h -generic-y += serial.h -generic-y += trace_clock.h generated-y += mach-types.h generated-y += unistd-nr.h diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index c815477b4303..413abfb42989 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -38,71 +38,6 @@ #define ICC_AP1R2 __ICC_AP1Rx(2) #define ICC_AP1R3 __ICC_AP1Rx(3) -#define ICC_HSRE __ACCESS_CP15(c12, 4, c9, 5) - -#define ICH_VSEIR __ACCESS_CP15(c12, 4, c9, 4) -#define ICH_HCR __ACCESS_CP15(c12, 4, c11, 0) -#define ICH_VTR __ACCESS_CP15(c12, 4, c11, 1) -#define ICH_MISR __ACCESS_CP15(c12, 4, c11, 2) -#define ICH_EISR __ACCESS_CP15(c12, 4, c11, 3) -#define ICH_ELRSR __ACCESS_CP15(c12, 4, c11, 5) -#define ICH_VMCR __ACCESS_CP15(c12, 4, c11, 7) - -#define __LR0(x) __ACCESS_CP15(c12, 4, c12, x) -#define __LR8(x) __ACCESS_CP15(c12, 4, c13, x) - -#define ICH_LR0 __LR0(0) -#define ICH_LR1 __LR0(1) -#define ICH_LR2 __LR0(2) -#define ICH_LR3 __LR0(3) -#define ICH_LR4 __LR0(4) -#define ICH_LR5 __LR0(5) -#define ICH_LR6 __LR0(6) -#define ICH_LR7 __LR0(7) -#define ICH_LR8 __LR8(0) -#define ICH_LR9 __LR8(1) -#define ICH_LR10 __LR8(2) -#define ICH_LR11 __LR8(3) -#define ICH_LR12 __LR8(4) -#define ICH_LR13 __LR8(5) -#define ICH_LR14 __LR8(6) -#define ICH_LR15 __LR8(7) - -/* LR top half */ -#define __LRC0(x) __ACCESS_CP15(c12, 4, c14, x) -#define __LRC8(x) __ACCESS_CP15(c12, 4, c15, x) - -#define ICH_LRC0 __LRC0(0) -#define ICH_LRC1 __LRC0(1) -#define ICH_LRC2 __LRC0(2) -#define ICH_LRC3 __LRC0(3) -#define ICH_LRC4 __LRC0(4) -#define ICH_LRC5 __LRC0(5) -#define ICH_LRC6 __LRC0(6) -#define ICH_LRC7 __LRC0(7) -#define ICH_LRC8 __LRC8(0) -#define ICH_LRC9 __LRC8(1) -#define ICH_LRC10 __LRC8(2) -#define ICH_LRC11 __LRC8(3) -#define ICH_LRC12 __LRC8(4) -#define ICH_LRC13 __LRC8(5) -#define ICH_LRC14 __LRC8(6) -#define ICH_LRC15 __LRC8(7) - -#define __ICH_AP0Rx(x) __ACCESS_CP15(c12, 4, c8, x) -#define ICH_AP0R0 __ICH_AP0Rx(0) -#define ICH_AP0R1 __ICH_AP0Rx(1) -#define ICH_AP0R2 __ICH_AP0Rx(2) -#define ICH_AP0R3 __ICH_AP0Rx(3) - -#define __ICH_AP1Rx(x) __ACCESS_CP15(c12, 4, c9, x) -#define ICH_AP1R0 __ICH_AP1Rx(0) -#define ICH_AP1R1 __ICH_AP1Rx(1) -#define ICH_AP1R2 __ICH_AP1Rx(2) -#define ICH_AP1R3 __ICH_AP1Rx(3) - -/* A32-to-A64 mappings used by VGIC save/restore */ - #define CPUIF_MAP(a32, a64) \ static inline void write_ ## a64(u32 val) \ { \ @@ -113,21 +48,6 @@ static inline u32 read_ ## a64(void) \ return read_sysreg(a32); \ } \ -#define CPUIF_MAP_LO_HI(a32lo, a32hi, a64) \ -static inline void write_ ## a64(u64 val) \ -{ \ - write_sysreg(lower_32_bits(val), a32lo);\ - write_sysreg(upper_32_bits(val), a32hi);\ -} \ -static inline u64 read_ ## a64(void) \ -{ \ - u64 val = read_sysreg(a32lo); \ - \ - val |= (u64)read_sysreg(a32hi) << 32; \ - \ - return val; \ -} - CPUIF_MAP(ICC_PMR, ICC_PMR_EL1) CPUIF_MAP(ICC_AP0R0, ICC_AP0R0_EL1) CPUIF_MAP(ICC_AP0R1, ICC_AP0R1_EL1) @@ -138,40 +58,6 @@ CPUIF_MAP(ICC_AP1R1, ICC_AP1R1_EL1) CPUIF_MAP(ICC_AP1R2, ICC_AP1R2_EL1) CPUIF_MAP(ICC_AP1R3, ICC_AP1R3_EL1) -CPUIF_MAP(ICH_HCR, ICH_HCR_EL2) -CPUIF_MAP(ICH_VTR, ICH_VTR_EL2) -CPUIF_MAP(ICH_MISR, ICH_MISR_EL2) -CPUIF_MAP(ICH_EISR, ICH_EISR_EL2) -CPUIF_MAP(ICH_ELRSR, ICH_ELRSR_EL2) -CPUIF_MAP(ICH_VMCR, ICH_VMCR_EL2) -CPUIF_MAP(ICH_AP0R3, ICH_AP0R3_EL2) -CPUIF_MAP(ICH_AP0R2, ICH_AP0R2_EL2) -CPUIF_MAP(ICH_AP0R1, ICH_AP0R1_EL2) -CPUIF_MAP(ICH_AP0R0, ICH_AP0R0_EL2) -CPUIF_MAP(ICH_AP1R3, ICH_AP1R3_EL2) -CPUIF_MAP(ICH_AP1R2, ICH_AP1R2_EL2) -CPUIF_MAP(ICH_AP1R1, ICH_AP1R1_EL2) -CPUIF_MAP(ICH_AP1R0, ICH_AP1R0_EL2) -CPUIF_MAP(ICC_HSRE, ICC_SRE_EL2) -CPUIF_MAP(ICC_SRE, ICC_SRE_EL1) - -CPUIF_MAP_LO_HI(ICH_LR15, ICH_LRC15, ICH_LR15_EL2) -CPUIF_MAP_LO_HI(ICH_LR14, ICH_LRC14, ICH_LR14_EL2) -CPUIF_MAP_LO_HI(ICH_LR13, ICH_LRC13, ICH_LR13_EL2) -CPUIF_MAP_LO_HI(ICH_LR12, ICH_LRC12, ICH_LR12_EL2) -CPUIF_MAP_LO_HI(ICH_LR11, ICH_LRC11, ICH_LR11_EL2) -CPUIF_MAP_LO_HI(ICH_LR10, ICH_LRC10, ICH_LR10_EL2) -CPUIF_MAP_LO_HI(ICH_LR9, ICH_LRC9, ICH_LR9_EL2) -CPUIF_MAP_LO_HI(ICH_LR8, ICH_LRC8, ICH_LR8_EL2) -CPUIF_MAP_LO_HI(ICH_LR7, ICH_LRC7, ICH_LR7_EL2) -CPUIF_MAP_LO_HI(ICH_LR6, ICH_LRC6, ICH_LR6_EL2) -CPUIF_MAP_LO_HI(ICH_LR5, ICH_LRC5, ICH_LR5_EL2) -CPUIF_MAP_LO_HI(ICH_LR4, ICH_LRC4, ICH_LR4_EL2) -CPUIF_MAP_LO_HI(ICH_LR3, ICH_LRC3, ICH_LR3_EL2) -CPUIF_MAP_LO_HI(ICH_LR2, ICH_LRC2, ICH_LR2_EL2) -CPUIF_MAP_LO_HI(ICH_LR1, ICH_LRC1, ICH_LR1_EL2) -CPUIF_MAP_LO_HI(ICH_LR0, ICH_LRC0, ICH_LR0_EL2) - #define read_gicreg(r) read_##r() #define write_gicreg(v, r) write_##r(v) diff --git a/arch/arm/include/asm/kvm_arm.h b/arch/arm/include/asm/kvm_arm.h deleted file mode 100644 index 9c04bd810d07..000000000000 --- a/arch/arm/include/asm/kvm_arm.h +++ /dev/null @@ -1,239 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#ifndef __ARM_KVM_ARM_H__ -#define __ARM_KVM_ARM_H__ - -#include -#include - -/* Hyp Configuration Register (HCR) bits */ -#define HCR_TGE (1 << 27) -#define HCR_TVM (1 << 26) -#define HCR_TTLB (1 << 25) -#define HCR_TPU (1 << 24) -#define HCR_TPC (1 << 23) -#define HCR_TSW (1 << 22) -#define HCR_TAC (1 << 21) -#define HCR_TIDCP (1 << 20) -#define HCR_TSC (1 << 19) -#define HCR_TID3 (1 << 18) -#define HCR_TID2 (1 << 17) -#define HCR_TID1 (1 << 16) -#define HCR_TID0 (1 << 15) -#define HCR_TWE (1 << 14) -#define HCR_TWI (1 << 13) -#define HCR_DC (1 << 12) -#define HCR_BSU (3 << 10) -#define HCR_BSU_IS (1 << 10) -#define HCR_FB (1 << 9) -#define HCR_VA (1 << 8) -#define HCR_VI (1 << 7) -#define HCR_VF (1 << 6) -#define HCR_AMO (1 << 5) -#define HCR_IMO (1 << 4) -#define HCR_FMO (1 << 3) -#define HCR_PTW (1 << 2) -#define HCR_SWIO (1 << 1) -#define HCR_VM 1 - -/* - * The bits we set in HCR: - * TAC: Trap ACTLR - * TSC: Trap SMC - * TVM: Trap VM ops (until MMU and caches are on) - * TSW: Trap cache operations by set/way - * TWI: Trap WFI - * TWE: Trap WFE - * TIDCP: Trap L2CTLR/L2ECTLR - * BSU_IS: Upgrade barriers to the inner shareable domain - * FB: Force broadcast of all maintainance operations - * AMO: Override CPSR.A and enable signaling with VA - * IMO: Override CPSR.I and enable signaling with VI - * FMO: Override CPSR.F and enable signaling with VF - * SWIO: Turn set/way invalidates into set/way clean+invalidate - */ -#define HCR_GUEST_MASK (HCR_TSC | HCR_TSW | HCR_TWI | HCR_VM | HCR_BSU_IS | \ - HCR_FB | HCR_TAC | HCR_AMO | HCR_IMO | HCR_FMO | \ - HCR_TVM | HCR_TWE | HCR_SWIO | HCR_TIDCP) - -/* System Control Register (SCTLR) bits */ -#define SCTLR_TE (1 << 30) -#define SCTLR_EE (1 << 25) -#define SCTLR_V (1 << 13) - -/* Hyp System Control Register (HSCTLR) bits */ -#define HSCTLR_TE (1 << 30) -#define HSCTLR_EE (1 << 25) -#define HSCTLR_FI (1 << 21) -#define HSCTLR_WXN (1 << 19) -#define HSCTLR_I (1 << 12) -#define HSCTLR_C (1 << 2) -#define HSCTLR_A (1 << 1) -#define HSCTLR_M 1 -#define HSCTLR_MASK (HSCTLR_M | HSCTLR_A | HSCTLR_C | HSCTLR_I | \ - HSCTLR_WXN | HSCTLR_FI | HSCTLR_EE | HSCTLR_TE) - -/* TTBCR and HTCR Registers bits */ -#define TTBCR_EAE (1 << 31) -#define TTBCR_IMP (1 << 30) -#define TTBCR_SH1 (3 << 28) -#define TTBCR_ORGN1 (3 << 26) -#define TTBCR_IRGN1 (3 << 24) -#define TTBCR_EPD1 (1 << 23) -#define TTBCR_A1 (1 << 22) -#define TTBCR_T1SZ (7 << 16) -#define TTBCR_SH0 (3 << 12) -#define TTBCR_ORGN0 (3 << 10) -#define TTBCR_IRGN0 (3 << 8) -#define TTBCR_EPD0 (1 << 7) -#define TTBCR_T0SZ (7 << 0) -#define HTCR_MASK (TTBCR_T0SZ | TTBCR_IRGN0 | TTBCR_ORGN0 | TTBCR_SH0) - -/* Hyp System Trap Register */ -#define HSTR_T(x) (1 << x) -#define HSTR_TTEE (1 << 16) -#define HSTR_TJDBX (1 << 17) - -/* Hyp Coprocessor Trap Register */ -#define HCPTR_TCP(x) (1 << x) -#define HCPTR_TCP_MASK (0x3fff) -#define HCPTR_TASE (1 << 15) -#define HCPTR_TTA (1 << 20) -#define HCPTR_TCPAC (1 << 31) - -/* Hyp Debug Configuration Register bits */ -#define HDCR_TDRA (1 << 11) -#define HDCR_TDOSA (1 << 10) -#define HDCR_TDA (1 << 9) -#define HDCR_TDE (1 << 8) -#define HDCR_HPME (1 << 7) -#define HDCR_TPM (1 << 6) -#define HDCR_TPMCR (1 << 5) -#define HDCR_HPMN_MASK (0x1F) - -/* - * The architecture supports 40-bit IPA as input to the 2nd stage translations - * and PTRS_PER_S2_PGD becomes 1024, because each entry covers 1GB of address - * space. - */ -#define KVM_PHYS_SHIFT (40) - -#define PTRS_PER_S2_PGD (_AC(1, ULL) << (KVM_PHYS_SHIFT - 30)) - -/* Virtualization Translation Control Register (VTCR) bits */ -#define VTCR_SH0 (3 << 12) -#define VTCR_ORGN0 (3 << 10) -#define VTCR_IRGN0 (3 << 8) -#define VTCR_SL0 (3 << 6) -#define VTCR_S (1 << 4) -#define VTCR_T0SZ (0xf) -#define VTCR_MASK (VTCR_SH0 | VTCR_ORGN0 | VTCR_IRGN0 | VTCR_SL0 | \ - VTCR_S | VTCR_T0SZ) -#define VTCR_HTCR_SH (VTCR_SH0 | VTCR_ORGN0 | VTCR_IRGN0) -#define VTCR_SL_L2 (0 << 6) /* Starting-level: 2 */ -#define VTCR_SL_L1 (1 << 6) /* Starting-level: 1 */ -#define KVM_VTCR_SL0 VTCR_SL_L1 -/* stage-2 input address range defined as 2^(32-T0SZ) */ -#define KVM_T0SZ (32 - KVM_PHYS_SHIFT) -#define KVM_VTCR_T0SZ (KVM_T0SZ & VTCR_T0SZ) -#define KVM_VTCR_S ((KVM_VTCR_T0SZ << 1) & VTCR_S) - -/* Virtualization Translation Table Base Register (VTTBR) bits */ -#if KVM_VTCR_SL0 == VTCR_SL_L2 /* see ARM DDI 0406C: B4-1720 */ -#define VTTBR_X (14 - KVM_T0SZ) -#else -#define VTTBR_X (5 - KVM_T0SZ) -#endif -#define VTTBR_CNP_BIT _AC(1, UL) -#define VTTBR_BADDR_MASK (((_AC(1, ULL) << (40 - VTTBR_X)) - 1) << VTTBR_X) -#define VTTBR_VMID_SHIFT _AC(48, ULL) -#define VTTBR_VMID_MASK(size) (_AT(u64, (1 << size) - 1) << VTTBR_VMID_SHIFT) - -/* Hyp Syndrome Register (HSR) bits */ -#define HSR_EC_SHIFT (26) -#define HSR_EC (_AC(0x3f, UL) << HSR_EC_SHIFT) -#define HSR_IL (_AC(1, UL) << 25) -#define HSR_ISS (HSR_IL - 1) -#define HSR_ISV_SHIFT (24) -#define HSR_ISV (_AC(1, UL) << HSR_ISV_SHIFT) -#define HSR_SRT_SHIFT (16) -#define HSR_SRT_MASK (0xf << HSR_SRT_SHIFT) -#define HSR_CM (1 << 8) -#define HSR_FSC (0x3f) -#define HSR_FSC_TYPE (0x3c) -#define HSR_SSE (1 << 21) -#define HSR_WNR (1 << 6) -#define HSR_CV_SHIFT (24) -#define HSR_CV (_AC(1, UL) << HSR_CV_SHIFT) -#define HSR_COND_SHIFT (20) -#define HSR_COND (_AC(0xf, UL) << HSR_COND_SHIFT) - -#define FSC_FAULT (0x04) -#define FSC_ACCESS (0x08) -#define FSC_PERM (0x0c) -#define FSC_SEA (0x10) -#define FSC_SEA_TTW0 (0x14) -#define FSC_SEA_TTW1 (0x15) -#define FSC_SEA_TTW2 (0x16) -#define FSC_SEA_TTW3 (0x17) -#define FSC_SECC (0x18) -#define FSC_SECC_TTW0 (0x1c) -#define FSC_SECC_TTW1 (0x1d) -#define FSC_SECC_TTW2 (0x1e) -#define FSC_SECC_TTW3 (0x1f) - -/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ -#define HPFAR_MASK (~0xf) - -#define HSR_EC_UNKNOWN (0x00) -#define HSR_EC_WFI (0x01) -#define HSR_EC_CP15_32 (0x03) -#define HSR_EC_CP15_64 (0x04) -#define HSR_EC_CP14_MR (0x05) -#define HSR_EC_CP14_LS (0x06) -#define HSR_EC_CP_0_13 (0x07) -#define HSR_EC_CP10_ID (0x08) -#define HSR_EC_JAZELLE (0x09) -#define HSR_EC_BXJ (0x0A) -#define HSR_EC_CP14_64 (0x0C) -#define HSR_EC_SVC_HYP (0x11) -#define HSR_EC_HVC (0x12) -#define HSR_EC_SMC (0x13) -#define HSR_EC_IABT (0x20) -#define HSR_EC_IABT_HYP (0x21) -#define HSR_EC_DABT (0x24) -#define HSR_EC_DABT_HYP (0x25) -#define HSR_EC_MAX (0x3f) - -#define HSR_WFI_IS_WFE (_AC(1, UL) << 0) - -#define HSR_HVC_IMM_MASK ((_AC(1, UL) << 16) - 1) - -#define HSR_DABT_S1PTW (_AC(1, UL) << 7) -#define HSR_DABT_CM (_AC(1, UL) << 8) - -#define kvm_arm_exception_type \ - {0, "RESET" }, \ - {1, "UNDEFINED" }, \ - {2, "SOFTWARE" }, \ - {3, "PREF_ABORT" }, \ - {4, "DATA_ABORT" }, \ - {5, "IRQ" }, \ - {6, "FIQ" }, \ - {7, "HVC" } - -#define HSRECN(x) { HSR_EC_##x, #x } - -#define kvm_arm_exception_class \ - HSRECN(UNKNOWN), HSRECN(WFI), HSRECN(CP15_32), HSRECN(CP15_64), \ - HSRECN(CP14_MR), HSRECN(CP14_LS), HSRECN(CP_0_13), HSRECN(CP10_ID), \ - HSRECN(JAZELLE), HSRECN(BXJ), HSRECN(CP14_64), HSRECN(SVC_HYP), \ - HSRECN(HVC), HSRECN(SMC), HSRECN(IABT), HSRECN(IABT_HYP), \ - HSRECN(DABT), HSRECN(DABT_HYP) - - -#endif /* __ARM_KVM_ARM_H__ */ diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h deleted file mode 100644 index f615830f9f57..000000000000 --- a/arch/arm/include/asm/kvm_asm.h +++ /dev/null @@ -1,77 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#ifndef __ARM_KVM_ASM_H__ -#define __ARM_KVM_ASM_H__ - -#include - -#define ARM_EXIT_WITH_ABORT_BIT 31 -#define ARM_EXCEPTION_CODE(x) ((x) & ~(1U << ARM_EXIT_WITH_ABORT_BIT)) -#define ARM_EXCEPTION_IS_TRAP(x) \ - (ARM_EXCEPTION_CODE((x)) == ARM_EXCEPTION_PREF_ABORT || \ - ARM_EXCEPTION_CODE((x)) == ARM_EXCEPTION_DATA_ABORT || \ - ARM_EXCEPTION_CODE((x)) == ARM_EXCEPTION_HVC) -#define ARM_ABORT_PENDING(x) !!((x) & (1U << ARM_EXIT_WITH_ABORT_BIT)) - -#define ARM_EXCEPTION_RESET 0 -#define ARM_EXCEPTION_UNDEFINED 1 -#define ARM_EXCEPTION_SOFTWARE 2 -#define ARM_EXCEPTION_PREF_ABORT 3 -#define ARM_EXCEPTION_DATA_ABORT 4 -#define ARM_EXCEPTION_IRQ 5 -#define ARM_EXCEPTION_FIQ 6 -#define ARM_EXCEPTION_HVC 7 -#define ARM_EXCEPTION_HYP_GONE HVC_STUB_ERR -/* - * The rr_lo_hi macro swaps a pair of registers depending on - * current endianness. It is used in conjunction with ldrd and strd - * instructions that load/store a 64-bit value from/to memory to/from - * a pair of registers which are used with the mrrc and mcrr instructions. - * If used with the ldrd/strd instructions, the a1 parameter is the first - * source/destination register and the a2 parameter is the second - * source/destination register. Note that the ldrd/strd instructions - * already swap the bytes within the words correctly according to the - * endianness setting, but the order of the registers need to be effectively - * swapped when used with the mrrc/mcrr instructions. - */ -#ifdef CONFIG_CPU_ENDIAN_BE8 -#define rr_lo_hi(a1, a2) a2, a1 -#else -#define rr_lo_hi(a1, a2) a1, a2 -#endif - -#define kvm_ksym_ref(kva) (kva) - -#ifndef __ASSEMBLY__ -struct kvm; -struct kvm_vcpu; - -extern char __kvm_hyp_init[]; -extern char __kvm_hyp_init_end[]; - -extern void __kvm_flush_vm_context(void); -extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); -extern void __kvm_tlb_flush_vmid(struct kvm *kvm); -extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu); - -extern void __kvm_timer_set_cntvoff(u32 cntvoff_low, u32 cntvoff_high); - -/* no VHE on 32-bit :( */ -static inline int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) { BUG(); return 0; } - -extern int __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu); - -extern void __init_stage2_translation(void); - -extern u64 __vgic_v3_get_ich_vtr_el2(void); -extern u64 __vgic_v3_read_vmcr(void); -extern void __vgic_v3_write_vmcr(u32 vmcr); -extern void __vgic_v3_init_lrs(void); - -#endif - -#endif /* __ARM_KVM_ASM_H__ */ diff --git a/arch/arm/include/asm/kvm_coproc.h b/arch/arm/include/asm/kvm_coproc.h deleted file mode 100644 index a23826117dd6..000000000000 --- a/arch/arm/include/asm/kvm_coproc.h +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 Rusty Russell IBM Corporation - */ - -#ifndef __ARM_KVM_COPROC_H__ -#define __ARM_KVM_COPROC_H__ -#include - -void kvm_reset_coprocs(struct kvm_vcpu *vcpu); - -struct kvm_coproc_target_table { - unsigned target; - const struct coproc_reg *table; - size_t num; -}; -void kvm_register_target_coproc_table(struct kvm_coproc_target_table *table); - -int kvm_handle_cp10_id(struct kvm_vcpu *vcpu, struct kvm_run *run); -int kvm_handle_cp_0_13_access(struct kvm_vcpu *vcpu, struct kvm_run *run); -int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run); -int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run); -int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run); -int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run); -int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run); - -unsigned long kvm_arm_num_guest_msrs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_msrindices(struct kvm_vcpu *vcpu, u64 __user *uindices); -void kvm_coproc_table_init(void); - -struct kvm_one_reg; -int kvm_arm_copy_coproc_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); -int kvm_arm_coproc_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *); -int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *); -unsigned long kvm_arm_num_coproc_regs(struct kvm_vcpu *vcpu); -#endif /* __ARM_KVM_COPROC_H__ */ diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h deleted file mode 100644 index 3944305e81df..000000000000 --- a/arch/arm/include/asm/kvm_emulate.h +++ /dev/null @@ -1,372 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#ifndef __ARM_KVM_EMULATE_H__ -#define __ARM_KVM_EMULATE_H__ - -#include -#include -#include -#include - -/* arm64 compatibility macros */ -#define PSR_AA32_MODE_FIQ FIQ_MODE -#define PSR_AA32_MODE_SVC SVC_MODE -#define PSR_AA32_MODE_ABT ABT_MODE -#define PSR_AA32_MODE_UND UND_MODE -#define PSR_AA32_T_BIT PSR_T_BIT -#define PSR_AA32_F_BIT PSR_F_BIT -#define PSR_AA32_I_BIT PSR_I_BIT -#define PSR_AA32_A_BIT PSR_A_BIT -#define PSR_AA32_E_BIT PSR_E_BIT -#define PSR_AA32_IT_MASK PSR_IT_MASK -#define PSR_AA32_GE_MASK 0x000f0000 -#define PSR_AA32_DIT_BIT 0x00200000 -#define PSR_AA32_PAN_BIT 0x00400000 -#define PSR_AA32_SSBS_BIT 0x00800000 -#define PSR_AA32_Q_BIT PSR_Q_BIT -#define PSR_AA32_V_BIT PSR_V_BIT -#define PSR_AA32_C_BIT PSR_C_BIT -#define PSR_AA32_Z_BIT PSR_Z_BIT -#define PSR_AA32_N_BIT PSR_N_BIT - -unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num); - -static inline unsigned long *vcpu_reg32(struct kvm_vcpu *vcpu, u8 reg_num) -{ - return vcpu_reg(vcpu, reg_num); -} - -unsigned long *__vcpu_spsr(struct kvm_vcpu *vcpu); - -static inline unsigned long vpcu_read_spsr(struct kvm_vcpu *vcpu) -{ - return *__vcpu_spsr(vcpu); -} - -static inline void vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long v) -{ - *__vcpu_spsr(vcpu) = v; -} - -static inline unsigned long host_spsr_to_spsr32(unsigned long spsr) -{ - return spsr; -} - -static inline unsigned long vcpu_get_reg(struct kvm_vcpu *vcpu, - u8 reg_num) -{ - return *vcpu_reg(vcpu, reg_num); -} - -static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num, - unsigned long val) -{ - *vcpu_reg(vcpu, reg_num) = val; -} - -bool kvm_condition_valid32(const struct kvm_vcpu *vcpu); -void kvm_skip_instr32(struct kvm_vcpu *vcpu, bool is_wide_instr); -void kvm_inject_undef32(struct kvm_vcpu *vcpu); -void kvm_inject_dabt32(struct kvm_vcpu *vcpu, unsigned long addr); -void kvm_inject_pabt32(struct kvm_vcpu *vcpu, unsigned long addr); -void kvm_inject_vabt(struct kvm_vcpu *vcpu); - -static inline void kvm_inject_undefined(struct kvm_vcpu *vcpu) -{ - kvm_inject_undef32(vcpu); -} - -static inline void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr) -{ - kvm_inject_dabt32(vcpu, addr); -} - -static inline void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr) -{ - kvm_inject_pabt32(vcpu, addr); -} - -static inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu) -{ - return kvm_condition_valid32(vcpu); -} - -static inline void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr) -{ - kvm_skip_instr32(vcpu, is_wide_instr); -} - -static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) -{ - vcpu->arch.hcr = HCR_GUEST_MASK; -} - -static inline unsigned long *vcpu_hcr(const struct kvm_vcpu *vcpu) -{ - return (unsigned long *)&vcpu->arch.hcr; -} - -static inline void vcpu_clear_wfx_traps(struct kvm_vcpu *vcpu) -{ - vcpu->arch.hcr &= ~HCR_TWE; -} - -static inline void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu) -{ - vcpu->arch.hcr |= HCR_TWE; -} - -static inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) -{ - return true; -} - -static inline unsigned long *vcpu_pc(struct kvm_vcpu *vcpu) -{ - return &vcpu->arch.ctxt.gp_regs.usr_regs.ARM_pc; -} - -static inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) -{ - return (unsigned long *)&vcpu->arch.ctxt.gp_regs.usr_regs.ARM_cpsr; -} - -static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) -{ - *vcpu_cpsr(vcpu) |= PSR_T_BIT; -} - -static inline bool mode_has_spsr(struct kvm_vcpu *vcpu) -{ - unsigned long cpsr_mode = vcpu->arch.ctxt.gp_regs.usr_regs.ARM_cpsr & MODE_MASK; - return (cpsr_mode > USR_MODE && cpsr_mode < SYSTEM_MODE); -} - -static inline bool vcpu_mode_priv(struct kvm_vcpu *vcpu) -{ - unsigned long cpsr_mode = vcpu->arch.ctxt.gp_regs.usr_regs.ARM_cpsr & MODE_MASK; - return cpsr_mode > USR_MODE; -} - -static inline u32 kvm_vcpu_get_hsr(const struct kvm_vcpu *vcpu) -{ - return vcpu->arch.fault.hsr; -} - -static inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu) -{ - u32 hsr = kvm_vcpu_get_hsr(vcpu); - - if (hsr & HSR_CV) - return (hsr & HSR_COND) >> HSR_COND_SHIFT; - - return -1; -} - -static inline unsigned long kvm_vcpu_get_hfar(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.fault.hxfar; -} - -static inline phys_addr_t kvm_vcpu_get_fault_ipa(struct kvm_vcpu *vcpu) -{ - return ((phys_addr_t)vcpu->arch.fault.hpfar & HPFAR_MASK) << 8; -} - -static inline bool kvm_vcpu_dabt_isvalid(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) & HSR_ISV; -} - -static inline unsigned long kvm_vcpu_dabt_iss_nisv_sanitized(const struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) & (HSR_CM | HSR_WNR | HSR_FSC); -} - -static inline bool kvm_vcpu_dabt_iswrite(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) & HSR_WNR; -} - -static inline bool kvm_vcpu_dabt_issext(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) & HSR_SSE; -} - -static inline bool kvm_vcpu_dabt_issf(const struct kvm_vcpu *vcpu) -{ - return false; -} - -static inline int kvm_vcpu_dabt_get_rd(struct kvm_vcpu *vcpu) -{ - return (kvm_vcpu_get_hsr(vcpu) & HSR_SRT_MASK) >> HSR_SRT_SHIFT; -} - -static inline bool kvm_vcpu_dabt_iss1tw(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) & HSR_DABT_S1PTW; -} - -static inline bool kvm_vcpu_dabt_is_cm(struct kvm_vcpu *vcpu) -{ - return !!(kvm_vcpu_get_hsr(vcpu) & HSR_DABT_CM); -} - -/* Get Access Size from a data abort */ -static inline unsigned int kvm_vcpu_dabt_get_as(struct kvm_vcpu *vcpu) -{ - switch ((kvm_vcpu_get_hsr(vcpu) >> 22) & 0x3) { - case 0: - return 1; - case 1: - return 2; - case 2: - return 4; - default: - kvm_err("Hardware is weird: SAS 0b11 is reserved\n"); - return 4; - } -} - -/* This one is not specific to Data Abort */ -static inline bool kvm_vcpu_trap_il_is32bit(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) & HSR_IL; -} - -static inline u8 kvm_vcpu_trap_get_class(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) >> HSR_EC_SHIFT; -} - -static inline bool kvm_vcpu_trap_is_iabt(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_trap_get_class(vcpu) == HSR_EC_IABT; -} - -static inline u8 kvm_vcpu_trap_get_fault(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) & HSR_FSC; -} - -static inline u8 kvm_vcpu_trap_get_fault_type(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) & HSR_FSC_TYPE; -} - -static inline bool kvm_vcpu_dabt_isextabt(struct kvm_vcpu *vcpu) -{ - switch (kvm_vcpu_trap_get_fault(vcpu)) { - case FSC_SEA: - case FSC_SEA_TTW0: - case FSC_SEA_TTW1: - case FSC_SEA_TTW2: - case FSC_SEA_TTW3: - case FSC_SECC: - case FSC_SECC_TTW0: - case FSC_SECC_TTW1: - case FSC_SECC_TTW2: - case FSC_SECC_TTW3: - return true; - default: - return false; - } -} - -static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) -{ - if (kvm_vcpu_trap_is_iabt(vcpu)) - return false; - - return kvm_vcpu_dabt_iswrite(vcpu); -} - -static inline u32 kvm_vcpu_hvc_get_imm(struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_get_hsr(vcpu) & HSR_HVC_IMM_MASK; -} - -static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) -{ - return vcpu_cp15(vcpu, c0_MPIDR) & MPIDR_HWID_BITMASK; -} - -static inline bool kvm_arm_get_vcpu_workaround_2_flag(struct kvm_vcpu *vcpu) -{ - return false; -} - -static inline void kvm_arm_set_vcpu_workaround_2_flag(struct kvm_vcpu *vcpu, - bool flag) -{ -} - -static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) -{ - *vcpu_cpsr(vcpu) |= PSR_E_BIT; -} - -static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) -{ - return !!(*vcpu_cpsr(vcpu) & PSR_E_BIT); -} - -static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu, - unsigned long data, - unsigned int len) -{ - if (kvm_vcpu_is_be(vcpu)) { - switch (len) { - case 1: - return data & 0xff; - case 2: - return be16_to_cpu(data & 0xffff); - default: - return be32_to_cpu(data); - } - } else { - switch (len) { - case 1: - return data & 0xff; - case 2: - return le16_to_cpu(data & 0xffff); - default: - return le32_to_cpu(data); - } - } -} - -static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu, - unsigned long data, - unsigned int len) -{ - if (kvm_vcpu_is_be(vcpu)) { - switch (len) { - case 1: - return data & 0xff; - case 2: - return cpu_to_be16(data & 0xffff); - default: - return cpu_to_be32(data); - } - } else { - switch (len) { - case 1: - return data & 0xff; - case 2: - return cpu_to_le16(data & 0xffff); - default: - return cpu_to_le32(data); - } - } -} - -static inline void vcpu_ptrauth_setup_lazy(struct kvm_vcpu *vcpu) {} - -#endif /* __ARM_KVM_EMULATE_H__ */ diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h deleted file mode 100644 index a827b4d60d38..000000000000 --- a/arch/arm/include/asm/kvm_host.h +++ /dev/null @@ -1,456 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#ifndef __ARM_KVM_HOST_H__ -#define __ARM_KVM_HOST_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define __KVM_HAVE_ARCH_INTC_INITIALIZED - -#define KVM_USER_MEM_SLOTS 32 -#define KVM_HAVE_ONE_REG -#define KVM_HALT_POLL_NS_DEFAULT 500000 - -#define KVM_VCPU_MAX_FEATURES 2 - -#include - - -#ifdef CONFIG_ARM_GIC_V3 -#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS -#else -#define KVM_MAX_VCPUS VGIC_V2_MAX_CPUS -#endif - -#define KVM_REQ_SLEEP \ - KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) -#define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) -#define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2) -#define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(3) - -DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use); - -static inline int kvm_arm_init_sve(void) { return 0; } - -u32 *kvm_vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num, u32 mode); -int __attribute_const__ kvm_target_cpu(void); -int kvm_reset_vcpu(struct kvm_vcpu *vcpu); -void kvm_reset_coprocs(struct kvm_vcpu *vcpu); - -struct kvm_vmid { - /* The VMID generation used for the virt. memory system */ - u64 vmid_gen; - u32 vmid; -}; - -struct kvm_arch { - /* The last vcpu id that ran on each physical CPU */ - int __percpu *last_vcpu_ran; - - /* - * Anything that is not used directly from assembly code goes - * here. - */ - - /* The VMID generation used for the virt. memory system */ - struct kvm_vmid vmid; - - /* Stage-2 page table */ - pgd_t *pgd; - phys_addr_t pgd_phys; - - /* Interrupt controller */ - struct vgic_dist vgic; - int max_vcpus; - - /* Mandated version of PSCI */ - u32 psci_version; - - /* - * If we encounter a data abort without valid instruction syndrome - * information, report this to user space. User space can (and - * should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is - * supported. - */ - bool return_nisv_io_abort_to_user; -}; - -#define KVM_NR_MEM_OBJS 40 - -/* - * We don't want allocation failures within the mmu code, so we preallocate - * enough memory for a single page fault in a cache. - */ -struct kvm_mmu_memory_cache { - int nobjs; - void *objects[KVM_NR_MEM_OBJS]; -}; - -struct kvm_vcpu_fault_info { - u32 hsr; /* Hyp Syndrome Register */ - u32 hxfar; /* Hyp Data/Inst. Fault Address Register */ - u32 hpfar; /* Hyp IPA Fault Address Register */ -}; - -/* - * 0 is reserved as an invalid value. - * Order should be kept in sync with the save/restore code. - */ -enum vcpu_sysreg { - __INVALID_SYSREG__, - c0_MPIDR, /* MultiProcessor ID Register */ - c0_CSSELR, /* Cache Size Selection Register */ - c1_SCTLR, /* System Control Register */ - c1_ACTLR, /* Auxiliary Control Register */ - c1_CPACR, /* Coprocessor Access Control */ - c2_TTBR0, /* Translation Table Base Register 0 */ - c2_TTBR0_high, /* TTBR0 top 32 bits */ - c2_TTBR1, /* Translation Table Base Register 1 */ - c2_TTBR1_high, /* TTBR1 top 32 bits */ - c2_TTBCR, /* Translation Table Base Control R. */ - c3_DACR, /* Domain Access Control Register */ - c5_DFSR, /* Data Fault Status Register */ - c5_IFSR, /* Instruction Fault Status Register */ - c5_ADFSR, /* Auxilary Data Fault Status R */ - c5_AIFSR, /* Auxilary Instrunction Fault Status R */ - c6_DFAR, /* Data Fault Address Register */ - c6_IFAR, /* Instruction Fault Address Register */ - c7_PAR, /* Physical Address Register */ - c7_PAR_high, /* PAR top 32 bits */ - c9_L2CTLR, /* Cortex A15/A7 L2 Control Register */ - c10_PRRR, /* Primary Region Remap Register */ - c10_NMRR, /* Normal Memory Remap Register */ - c12_VBAR, /* Vector Base Address Register */ - c13_CID, /* Context ID Register */ - c13_TID_URW, /* Thread ID, User R/W */ - c13_TID_URO, /* Thread ID, User R/O */ - c13_TID_PRIV, /* Thread ID, Privileged */ - c14_CNTKCTL, /* Timer Control Register (PL1) */ - c10_AMAIR0, /* Auxilary Memory Attribute Indirection Reg0 */ - c10_AMAIR1, /* Auxilary Memory Attribute Indirection Reg1 */ - NR_CP15_REGS /* Number of regs (incl. invalid) */ -}; - -struct kvm_cpu_context { - struct kvm_regs gp_regs; - struct vfp_hard_struct vfp; - u32 cp15[NR_CP15_REGS]; -}; - -struct kvm_host_data { - struct kvm_cpu_context host_ctxt; -}; - -typedef struct kvm_host_data kvm_host_data_t; - -static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt) -{ - /* The host's MPIDR is immutable, so let's set it up at boot time */ - cpu_ctxt->cp15[c0_MPIDR] = read_cpuid_mpidr(); -} - -struct vcpu_reset_state { - unsigned long pc; - unsigned long r0; - bool be; - bool reset; -}; - -struct kvm_vcpu_arch { - struct kvm_cpu_context ctxt; - - int target; /* Processor target */ - DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES); - - /* The CPU type we expose to the VM */ - u32 midr; - - /* HYP trapping configuration */ - u32 hcr; - - /* Exception Information */ - struct kvm_vcpu_fault_info fault; - - /* Host FP context */ - struct kvm_cpu_context *host_cpu_context; - - /* VGIC state */ - struct vgic_cpu vgic_cpu; - struct arch_timer_cpu timer_cpu; - - /* - * Anything that is not used directly from assembly code goes - * here. - */ - - /* vcpu power-off state */ - bool power_off; - - /* Don't run the guest (internal implementation need) */ - bool pause; - - /* Cache some mmu pages needed inside spinlock regions */ - struct kvm_mmu_memory_cache mmu_page_cache; - - struct vcpu_reset_state reset_state; - - /* Detect first run of a vcpu */ - bool has_run_once; -}; - -struct kvm_vm_stat { - ulong remote_tlb_flush; -}; - -struct kvm_vcpu_stat { - u64 halt_successful_poll; - u64 halt_attempted_poll; - u64 halt_poll_invalid; - u64 halt_wakeup; - u64 hvc_exit_stat; - u64 wfe_exit_stat; - u64 wfi_exit_stat; - u64 mmio_exit_user; - u64 mmio_exit_kernel; - u64 exits; -}; - -#define vcpu_cp15(v,r) (v)->arch.ctxt.cp15[r] - -int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init); -unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); -int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); -int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); - -unsigned long __kvm_call_hyp(void *hypfn, ...); - -/* - * The has_vhe() part doesn't get emitted, but is used for type-checking. - */ -#define kvm_call_hyp(f, ...) \ - do { \ - if (has_vhe()) { \ - f(__VA_ARGS__); \ - } else { \ - __kvm_call_hyp(kvm_ksym_ref(f), ##__VA_ARGS__); \ - } \ - } while(0) - -#define kvm_call_hyp_ret(f, ...) \ - ({ \ - typeof(f(__VA_ARGS__)) ret; \ - \ - if (has_vhe()) { \ - ret = f(__VA_ARGS__); \ - } else { \ - ret = __kvm_call_hyp(kvm_ksym_ref(f), \ - ##__VA_ARGS__); \ - } \ - \ - ret; \ - }) - -void force_vm_exit(const cpumask_t *mask); -int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, - struct kvm_vcpu_events *events); - -int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, - struct kvm_vcpu_events *events); - -#define KVM_ARCH_WANT_MMU_NOTIFIER -int kvm_unmap_hva_range(struct kvm *kvm, - unsigned long start, unsigned long end); -int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); - -unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); -int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); -int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); - -void kvm_arm_halt_guest(struct kvm *kvm); -void kvm_arm_resume_guest(struct kvm *kvm); - -int kvm_arm_copy_coproc_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); -unsigned long kvm_arm_num_coproc_regs(struct kvm_vcpu *vcpu); -int kvm_arm_coproc_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *); -int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *); - -int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, - int exception_index); - -static inline void handle_exit_early(struct kvm_vcpu *vcpu, struct kvm_run *run, - int exception_index) {} - -/* MMIO helpers */ -void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data); -unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len); - -int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); -int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, - phys_addr_t fault_ipa); - -static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, - unsigned long hyp_stack_ptr, - unsigned long vector_ptr) -{ - /* - * Call initialization code, and switch to the full blown HYP - * code. The init code doesn't need to preserve these - * registers as r0-r3 are already callee saved according to - * the AAPCS. - * Note that we slightly misuse the prototype by casting the - * stack pointer to a void *. - - * The PGDs are always passed as the third argument, in order - * to be passed into r2-r3 to the init code (yes, this is - * compliant with the PCS!). - */ - - __kvm_call_hyp((void*)hyp_stack_ptr, vector_ptr, pgd_ptr); -} - -static inline void __cpu_init_stage2(void) -{ - kvm_call_hyp(__init_stage2_translation); -} - -static inline int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext) -{ - return 0; -} - -int kvm_perf_init(void); -int kvm_perf_teardown(void); - -static inline long kvm_hypercall_pv_features(struct kvm_vcpu *vcpu) -{ - return SMCCC_RET_NOT_SUPPORTED; -} - -static inline gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu) -{ - return GPA_INVALID; -} - -static inline void kvm_update_stolen_time(struct kvm_vcpu *vcpu) -{ -} - -static inline void kvm_arm_pvtime_vcpu_init(struct kvm_vcpu_arch *vcpu_arch) -{ -} - -static inline bool kvm_arm_is_pvtime_enabled(struct kvm_vcpu_arch *vcpu_arch) -{ - return false; -} - -void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot); - -struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr); - -static inline bool kvm_arch_requires_vhe(void) { return false; } -static inline void kvm_arch_hardware_unsetup(void) {} -static inline void kvm_arch_sync_events(struct kvm *kvm) {} -static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} -static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} -static inline void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu) {} - -static inline void kvm_arm_init_debug(void) {} -static inline void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) {} -static inline void kvm_arm_clear_debug(struct kvm_vcpu *vcpu) {} -static inline void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu) {} - -int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, - struct kvm_device_attr *attr); -int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, - struct kvm_device_attr *attr); -int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, - struct kvm_device_attr *attr); - -/* - * VFP/NEON switching is all done by the hyp switch code, so no need to - * coordinate with host context handling for this state: - */ -static inline void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) {} -static inline void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) {} -static inline void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) {} - -static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {} -static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {} - -#define KVM_BP_HARDEN_UNKNOWN -1 -#define KVM_BP_HARDEN_WA_NEEDED 0 -#define KVM_BP_HARDEN_NOT_REQUIRED 1 - -static inline int kvm_arm_harden_branch_predictor(void) -{ - switch(read_cpuid_part()) { -#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR - case ARM_CPU_PART_BRAHMA_B15: - case ARM_CPU_PART_CORTEX_A12: - case ARM_CPU_PART_CORTEX_A15: - case ARM_CPU_PART_CORTEX_A17: - return KVM_BP_HARDEN_WA_NEEDED; -#endif - case ARM_CPU_PART_CORTEX_A7: - return KVM_BP_HARDEN_NOT_REQUIRED; - default: - return KVM_BP_HARDEN_UNKNOWN; - } -} - -#define KVM_SSBD_UNKNOWN -1 -#define KVM_SSBD_FORCE_DISABLE 0 -#define KVM_SSBD_KERNEL 1 -#define KVM_SSBD_FORCE_ENABLE 2 -#define KVM_SSBD_MITIGATED 3 - -static inline int kvm_arm_have_ssbd(void) -{ - /* No way to detect it yet, pretend it is not there. */ - return KVM_SSBD_UNKNOWN; -} - -static inline void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu) {} -static inline void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu) {} - -#define __KVM_HAVE_ARCH_VM_ALLOC -struct kvm *kvm_arch_alloc_vm(void); -void kvm_arch_free_vm(struct kvm *kvm); - -static inline int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type) -{ - /* - * On 32bit ARM, VMs get a static 40bit IPA stage2 setup, - * so any non-zero value used as type is illegal. - */ - if (type) - return -EINVAL; - return 0; -} - -static inline int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature) -{ - return -EINVAL; -} - -static inline bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu) -{ - return true; -} - -#endif /* __ARM_KVM_HOST_H__ */ diff --git a/arch/arm/include/asm/kvm_hyp.h b/arch/arm/include/asm/kvm_hyp.h deleted file mode 100644 index 3c1b55ecc578..000000000000 --- a/arch/arm/include/asm/kvm_hyp.h +++ /dev/null @@ -1,127 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2015 - ARM Ltd - * Author: Marc Zyngier - */ - -#ifndef __ARM_KVM_HYP_H__ -#define __ARM_KVM_HYP_H__ - -#include -#include -#include -#include -#include - -#define __hyp_text __section(.hyp.text) notrace - -#define __ACCESS_VFP(CRn) \ - "mrc", "mcr", __stringify(p10, 7, %0, CRn, cr0, 0), u32 - -#define write_special(v, r) \ - asm volatile("msr " __stringify(r) ", %0" : : "r" (v)) -#define read_special(r) ({ \ - u32 __val; \ - asm volatile("mrs %0, " __stringify(r) : "=r" (__val)); \ - __val; \ -}) - -#define TTBR0 __ACCESS_CP15_64(0, c2) -#define TTBR1 __ACCESS_CP15_64(1, c2) -#define VTTBR __ACCESS_CP15_64(6, c2) -#define PAR __ACCESS_CP15_64(0, c7) -#define CNTP_CVAL __ACCESS_CP15_64(2, c14) -#define CNTV_CVAL __ACCESS_CP15_64(3, c14) -#define CNTVOFF __ACCESS_CP15_64(4, c14) - -#define MIDR __ACCESS_CP15(c0, 0, c0, 0) -#define CSSELR __ACCESS_CP15(c0, 2, c0, 0) -#define VPIDR __ACCESS_CP15(c0, 4, c0, 0) -#define VMPIDR __ACCESS_CP15(c0, 4, c0, 5) -#define SCTLR __ACCESS_CP15(c1, 0, c0, 0) -#define CPACR __ACCESS_CP15(c1, 0, c0, 2) -#define HCR __ACCESS_CP15(c1, 4, c1, 0) -#define HDCR __ACCESS_CP15(c1, 4, c1, 1) -#define HCPTR __ACCESS_CP15(c1, 4, c1, 2) -#define HSTR __ACCESS_CP15(c1, 4, c1, 3) -#define TTBCR __ACCESS_CP15(c2, 0, c0, 2) -#define HTCR __ACCESS_CP15(c2, 4, c0, 2) -#define VTCR __ACCESS_CP15(c2, 4, c1, 2) -#define DACR __ACCESS_CP15(c3, 0, c0, 0) -#define DFSR __ACCESS_CP15(c5, 0, c0, 0) -#define IFSR __ACCESS_CP15(c5, 0, c0, 1) -#define ADFSR __ACCESS_CP15(c5, 0, c1, 0) -#define AIFSR __ACCESS_CP15(c5, 0, c1, 1) -#define HSR __ACCESS_CP15(c5, 4, c2, 0) -#define DFAR __ACCESS_CP15(c6, 0, c0, 0) -#define IFAR __ACCESS_CP15(c6, 0, c0, 2) -#define HDFAR __ACCESS_CP15(c6, 4, c0, 0) -#define HIFAR __ACCESS_CP15(c6, 4, c0, 2) -#define HPFAR __ACCESS_CP15(c6, 4, c0, 4) -#define ICIALLUIS __ACCESS_CP15(c7, 0, c1, 0) -#define BPIALLIS __ACCESS_CP15(c7, 0, c1, 6) -#define ICIMVAU __ACCESS_CP15(c7, 0, c5, 1) -#define ATS1CPR __ACCESS_CP15(c7, 0, c8, 0) -#define TLBIALLIS __ACCESS_CP15(c8, 0, c3, 0) -#define TLBIALL __ACCESS_CP15(c8, 0, c7, 0) -#define TLBIALLNSNHIS __ACCESS_CP15(c8, 4, c3, 4) -#define PRRR __ACCESS_CP15(c10, 0, c2, 0) -#define NMRR __ACCESS_CP15(c10, 0, c2, 1) -#define AMAIR0 __ACCESS_CP15(c10, 0, c3, 0) -#define AMAIR1 __ACCESS_CP15(c10, 0, c3, 1) -#define VBAR __ACCESS_CP15(c12, 0, c0, 0) -#define CID __ACCESS_CP15(c13, 0, c0, 1) -#define TID_URW __ACCESS_CP15(c13, 0, c0, 2) -#define TID_URO __ACCESS_CP15(c13, 0, c0, 3) -#define TID_PRIV __ACCESS_CP15(c13, 0, c0, 4) -#define HTPIDR __ACCESS_CP15(c13, 4, c0, 2) -#define CNTKCTL __ACCESS_CP15(c14, 0, c1, 0) -#define CNTP_CTL __ACCESS_CP15(c14, 0, c2, 1) -#define CNTV_CTL __ACCESS_CP15(c14, 0, c3, 1) -#define CNTHCTL __ACCESS_CP15(c14, 4, c1, 0) - -#define VFP_FPEXC __ACCESS_VFP(FPEXC) - -/* AArch64 compatibility macros, only for the timer so far */ -#define read_sysreg_el0(r) read_sysreg(r##_EL0) -#define write_sysreg_el0(v, r) write_sysreg(v, r##_EL0) - -#define SYS_CNTP_CTL_EL0 CNTP_CTL -#define SYS_CNTP_CVAL_EL0 CNTP_CVAL -#define SYS_CNTV_CTL_EL0 CNTV_CTL -#define SYS_CNTV_CVAL_EL0 CNTV_CVAL - -#define cntvoff_el2 CNTVOFF -#define cnthctl_el2 CNTHCTL - -void __timer_enable_traps(struct kvm_vcpu *vcpu); -void __timer_disable_traps(struct kvm_vcpu *vcpu); - -void __vgic_v2_save_state(struct kvm_vcpu *vcpu); -void __vgic_v2_restore_state(struct kvm_vcpu *vcpu); - -void __sysreg_save_state(struct kvm_cpu_context *ctxt); -void __sysreg_restore_state(struct kvm_cpu_context *ctxt); - -void __vgic_v3_save_state(struct kvm_vcpu *vcpu); -void __vgic_v3_restore_state(struct kvm_vcpu *vcpu); -void __vgic_v3_activate_traps(struct kvm_vcpu *vcpu); -void __vgic_v3_deactivate_traps(struct kvm_vcpu *vcpu); -void __vgic_v3_save_aprs(struct kvm_vcpu *vcpu); -void __vgic_v3_restore_aprs(struct kvm_vcpu *vcpu); - -asmlinkage void __vfp_save_state(struct vfp_hard_struct *vfp); -asmlinkage void __vfp_restore_state(struct vfp_hard_struct *vfp); -static inline bool __vfp_enabled(void) -{ - return !(read_sysreg(HCPTR) & (HCPTR_TCP(11) | HCPTR_TCP(10))); -} - -void __hyp_text __banked_save_state(struct kvm_cpu_context *ctxt); -void __hyp_text __banked_restore_state(struct kvm_cpu_context *ctxt); - -asmlinkage int __guest_enter(struct kvm_vcpu *vcpu, - struct kvm_cpu_context *host); -asmlinkage int __hyp_do_panic(const char *, int, u32); - -#endif /* __ARM_KVM_HYP_H__ */ diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h deleted file mode 100644 index 0d84d50bf9ba..000000000000 --- a/arch/arm/include/asm/kvm_mmu.h +++ /dev/null @@ -1,435 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#ifndef __ARM_KVM_MMU_H__ -#define __ARM_KVM_MMU_H__ - -#include -#include - -/* - * We directly use the kernel VA for the HYP, as we can directly share - * the mapping (HTTBR "covers" TTBR1). - */ -#define kern_hyp_va(kva) (kva) - -/* Contrary to arm64, there is no need to generate a PC-relative address */ -#define hyp_symbol_addr(s) \ - ({ \ - typeof(s) *addr = &(s); \ - addr; \ - }) - -#ifndef __ASSEMBLY__ - -#include -#include -#include -#include -#include -#include -#include - -/* Ensure compatibility with arm64 */ -#define VA_BITS 32 - -#define kvm_phys_shift(kvm) KVM_PHYS_SHIFT -#define kvm_phys_size(kvm) (1ULL << kvm_phys_shift(kvm)) -#define kvm_phys_mask(kvm) (kvm_phys_size(kvm) - 1ULL) -#define kvm_vttbr_baddr_mask(kvm) VTTBR_BADDR_MASK - -#define stage2_pgd_size(kvm) (PTRS_PER_S2_PGD * sizeof(pgd_t)) - -int create_hyp_mappings(void *from, void *to, pgprot_t prot); -int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, - void __iomem **kaddr, - void __iomem **haddr); -int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size, - void **haddr); -void free_hyp_pgds(void); - -void stage2_unmap_vm(struct kvm *kvm); -int kvm_alloc_stage2_pgd(struct kvm *kvm); -void kvm_free_stage2_pgd(struct kvm *kvm); -int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, - phys_addr_t pa, unsigned long size, bool writable); - -int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run); - -void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); - -phys_addr_t kvm_mmu_get_httbr(void); -phys_addr_t kvm_get_idmap_vector(void); -int kvm_mmu_init(void); -void kvm_clear_hyp_idmap(void); - -#define kvm_mk_pmd(ptep) __pmd(__pa(ptep) | PMD_TYPE_TABLE) -#define kvm_mk_pud(pmdp) __pud(__pa(pmdp) | PMD_TYPE_TABLE) -#define kvm_mk_pgd(pudp) ({ BUILD_BUG(); 0; }) - -#define kvm_pfn_pte(pfn, prot) pfn_pte(pfn, prot) -#define kvm_pfn_pmd(pfn, prot) pfn_pmd(pfn, prot) -#define kvm_pfn_pud(pfn, prot) (__pud(0)) - -#define kvm_pud_pfn(pud) ({ WARN_ON(1); 0; }) - - -#define kvm_pmd_mkhuge(pmd) pmd_mkhuge(pmd) -/* No support for pud hugepages */ -#define kvm_pud_mkhuge(pud) ( {WARN_ON(1); pud; }) - -/* - * The following kvm_*pud*() functions are provided strictly to allow - * sharing code with arm64. They should never be called in practice. - */ -static inline void kvm_set_s2pud_readonly(pud_t *pud) -{ - WARN_ON(1); -} - -static inline bool kvm_s2pud_readonly(pud_t *pud) -{ - WARN_ON(1); - return false; -} - -static inline void kvm_set_pud(pud_t *pud, pud_t new_pud) -{ - WARN_ON(1); -} - -static inline pud_t kvm_s2pud_mkwrite(pud_t pud) -{ - WARN_ON(1); - return pud; -} - -static inline pud_t kvm_s2pud_mkexec(pud_t pud) -{ - WARN_ON(1); - return pud; -} - -static inline bool kvm_s2pud_exec(pud_t *pud) -{ - WARN_ON(1); - return false; -} - -static inline pud_t kvm_s2pud_mkyoung(pud_t pud) -{ - BUG(); - return pud; -} - -static inline bool kvm_s2pud_young(pud_t pud) -{ - WARN_ON(1); - return false; -} - -static inline pte_t kvm_s2pte_mkwrite(pte_t pte) -{ - pte_val(pte) |= L_PTE_S2_RDWR; - return pte; -} - -static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd) -{ - pmd_val(pmd) |= L_PMD_S2_RDWR; - return pmd; -} - -static inline pte_t kvm_s2pte_mkexec(pte_t pte) -{ - pte_val(pte) &= ~L_PTE_XN; - return pte; -} - -static inline pmd_t kvm_s2pmd_mkexec(pmd_t pmd) -{ - pmd_val(pmd) &= ~PMD_SECT_XN; - return pmd; -} - -static inline void kvm_set_s2pte_readonly(pte_t *pte) -{ - pte_val(*pte) = (pte_val(*pte) & ~L_PTE_S2_RDWR) | L_PTE_S2_RDONLY; -} - -static inline bool kvm_s2pte_readonly(pte_t *pte) -{ - return (pte_val(*pte) & L_PTE_S2_RDWR) == L_PTE_S2_RDONLY; -} - -static inline bool kvm_s2pte_exec(pte_t *pte) -{ - return !(pte_val(*pte) & L_PTE_XN); -} - -static inline void kvm_set_s2pmd_readonly(pmd_t *pmd) -{ - pmd_val(*pmd) = (pmd_val(*pmd) & ~L_PMD_S2_RDWR) | L_PMD_S2_RDONLY; -} - -static inline bool kvm_s2pmd_readonly(pmd_t *pmd) -{ - return (pmd_val(*pmd) & L_PMD_S2_RDWR) == L_PMD_S2_RDONLY; -} - -static inline bool kvm_s2pmd_exec(pmd_t *pmd) -{ - return !(pmd_val(*pmd) & PMD_SECT_XN); -} - -static inline bool kvm_page_empty(void *ptr) -{ - struct page *ptr_page = virt_to_page(ptr); - return page_count(ptr_page) == 1; -} - -#define kvm_pte_table_empty(kvm, ptep) kvm_page_empty(ptep) -#define kvm_pmd_table_empty(kvm, pmdp) kvm_page_empty(pmdp) -#define kvm_pud_table_empty(kvm, pudp) false - -#define hyp_pte_table_empty(ptep) kvm_page_empty(ptep) -#define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp) -#define hyp_pud_table_empty(pudp) false - -struct kvm; - -#define kvm_flush_dcache_to_poc(a,l) __cpuc_flush_dcache_area((a), (l)) - -static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu) -{ - return (vcpu_cp15(vcpu, c1_SCTLR) & 0b101) == 0b101; -} - -static inline void __clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size) -{ - /* - * Clean the dcache to the Point of Coherency. - * - * We need to do this through a kernel mapping (using the - * user-space mapping has proved to be the wrong - * solution). For that, we need to kmap one page at a time, - * and iterate over the range. - */ - - VM_BUG_ON(size & ~PAGE_MASK); - - while (size) { - void *va = kmap_atomic_pfn(pfn); - - kvm_flush_dcache_to_poc(va, PAGE_SIZE); - - size -= PAGE_SIZE; - pfn++; - - kunmap_atomic(va); - } -} - -static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn, - unsigned long size) -{ - u32 iclsz; - - /* - * If we are going to insert an instruction page and the icache is - * either VIPT or PIPT, there is a potential problem where the host - * (or another VM) may have used the same page as this guest, and we - * read incorrect data from the icache. If we're using a PIPT cache, - * we can invalidate just that page, but if we are using a VIPT cache - * we need to invalidate the entire icache - damn shame - as written - * in the ARM ARM (DDI 0406C.b - Page B3-1393). - * - * VIVT caches are tagged using both the ASID and the VMID and doesn't - * need any kind of flushing (DDI 0406C.b - Page B3-1392). - */ - - VM_BUG_ON(size & ~PAGE_MASK); - - if (icache_is_vivt_asid_tagged()) - return; - - if (!icache_is_pipt()) { - /* any kind of VIPT cache */ - __flush_icache_all(); - return; - } - - /* - * CTR IminLine contains Log2 of the number of words in the - * cache line, so we can get the number of words as - * 2 << (IminLine - 1). To get the number of bytes, we - * multiply by 4 (the number of bytes in a 32-bit word), and - * get 4 << (IminLine). - */ - iclsz = 4 << (read_cpuid(CPUID_CACHETYPE) & 0xf); - - while (size) { - void *va = kmap_atomic_pfn(pfn); - void *end = va + PAGE_SIZE; - void *addr = va; - - do { - write_sysreg(addr, ICIMVAU); - addr += iclsz; - } while (addr < end); - - dsb(ishst); - isb(); - - size -= PAGE_SIZE; - pfn++; - - kunmap_atomic(va); - } - - /* Check if we need to invalidate the BTB */ - if ((read_cpuid_ext(CPUID_EXT_MMFR1) >> 28) != 4) { - write_sysreg(0, BPIALLIS); - dsb(ishst); - isb(); - } -} - -static inline void __kvm_flush_dcache_pte(pte_t pte) -{ - void *va = kmap_atomic(pte_page(pte)); - - kvm_flush_dcache_to_poc(va, PAGE_SIZE); - - kunmap_atomic(va); -} - -static inline void __kvm_flush_dcache_pmd(pmd_t pmd) -{ - unsigned long size = PMD_SIZE; - kvm_pfn_t pfn = pmd_pfn(pmd); - - while (size) { - void *va = kmap_atomic_pfn(pfn); - - kvm_flush_dcache_to_poc(va, PAGE_SIZE); - - pfn++; - size -= PAGE_SIZE; - - kunmap_atomic(va); - } -} - -static inline void __kvm_flush_dcache_pud(pud_t pud) -{ -} - -#define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x)) - -void kvm_set_way_flush(struct kvm_vcpu *vcpu); -void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled); - -static inline bool __kvm_cpu_uses_extended_idmap(void) -{ - return false; -} - -static inline unsigned long __kvm_idmap_ptrs_per_pgd(void) -{ - return PTRS_PER_PGD; -} - -static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd, - pgd_t *hyp_pgd, - pgd_t *merged_hyp_pgd, - unsigned long hyp_idmap_start) { } - -static inline unsigned int kvm_get_vmid_bits(void) -{ - return 8; -} - -/* - * We are not in the kvm->srcu critical section most of the time, so we take - * the SRCU read lock here. Since we copy the data from the user page, we - * can immediately drop the lock again. - */ -static inline int kvm_read_guest_lock(struct kvm *kvm, - gpa_t gpa, void *data, unsigned long len) -{ - int srcu_idx = srcu_read_lock(&kvm->srcu); - int ret = kvm_read_guest(kvm, gpa, data, len); - - srcu_read_unlock(&kvm->srcu, srcu_idx); - - return ret; -} - -static inline int kvm_write_guest_lock(struct kvm *kvm, gpa_t gpa, - const void *data, unsigned long len) -{ - int srcu_idx = srcu_read_lock(&kvm->srcu); - int ret = kvm_write_guest(kvm, gpa, data, len); - - srcu_read_unlock(&kvm->srcu, srcu_idx); - - return ret; -} - -static inline void *kvm_get_hyp_vector(void) -{ - switch(read_cpuid_part()) { -#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR - case ARM_CPU_PART_CORTEX_A12: - case ARM_CPU_PART_CORTEX_A17: - { - extern char __kvm_hyp_vector_bp_inv[]; - return kvm_ksym_ref(__kvm_hyp_vector_bp_inv); - } - - case ARM_CPU_PART_BRAHMA_B15: - case ARM_CPU_PART_CORTEX_A15: - { - extern char __kvm_hyp_vector_ic_inv[]; - return kvm_ksym_ref(__kvm_hyp_vector_ic_inv); - } -#endif - default: - { - extern char __kvm_hyp_vector[]; - return kvm_ksym_ref(__kvm_hyp_vector); - } - } -} - -static inline int kvm_map_vectors(void) -{ - return 0; -} - -static inline int hyp_map_aux_data(void) -{ - return 0; -} - -#define kvm_phys_to_vttbr(addr) (addr) - -static inline void kvm_set_ipa_limit(void) {} - -static __always_inline u64 kvm_get_vttbr(struct kvm *kvm) -{ - struct kvm_vmid *vmid = &kvm->arch.vmid; - u64 vmid_field, baddr; - - baddr = kvm->arch.pgd_phys; - vmid_field = (u64)vmid->vmid << VTTBR_VMID_SHIFT; - return kvm_phys_to_vttbr(baddr) | vmid_field; -} - -#endif /* !__ASSEMBLY__ */ - -#endif /* __ARM_KVM_MMU_H__ */ diff --git a/arch/arm/include/asm/kvm_ras.h b/arch/arm/include/asm/kvm_ras.h deleted file mode 100644 index e9577292dfe4..000000000000 --- a/arch/arm/include/asm/kvm_ras.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2018 - Arm Ltd */ - -#ifndef __ARM_KVM_RAS_H__ -#define __ARM_KVM_RAS_H__ - -#include - -static inline int kvm_handle_guest_sea(phys_addr_t addr, unsigned int esr) -{ - return -1; -} - -#endif /* __ARM_KVM_RAS_H__ */ diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index ad55ab068dbf..36805f94939e 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -104,26 +104,6 @@ */ #define L_PGD_SWAPPER (_AT(pgdval_t, 1) << 55) /* swapper_pg_dir entry */ -/* - * 2nd stage PTE definitions for LPAE. - */ -#define L_PTE_S2_MT_UNCACHED (_AT(pteval_t, 0x0) << 2) /* strongly ordered */ -#define L_PTE_S2_MT_WRITETHROUGH (_AT(pteval_t, 0xa) << 2) /* normal inner write-through */ -#define L_PTE_S2_MT_WRITEBACK (_AT(pteval_t, 0xf) << 2) /* normal inner write-back */ -#define L_PTE_S2_MT_DEV_SHARED (_AT(pteval_t, 0x1) << 2) /* device */ -#define L_PTE_S2_MT_MASK (_AT(pteval_t, 0xf) << 2) - -#define L_PTE_S2_RDONLY (_AT(pteval_t, 1) << 6) /* HAP[1] */ -#define L_PTE_S2_RDWR (_AT(pteval_t, 3) << 6) /* HAP[2:1] */ - -#define L_PMD_S2_RDONLY (_AT(pmdval_t, 1) << 6) /* HAP[1] */ -#define L_PMD_S2_RDWR (_AT(pmdval_t, 3) << 6) /* HAP[2:1] */ - -/* - * Hyp-mode PL2 PTE definitions for LPAE. - */ -#define L_PTE_HYP L_PTE_USER - #ifndef __ASSEMBLY__ #define pud_none(pud) (!pud_val(pud)) diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index eabcb48a7840..0483cf413315 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -80,9 +80,6 @@ extern void __pgd_error(const char *file, int line, pgd_t); extern pgprot_t pgprot_user; extern pgprot_t pgprot_kernel; -extern pgprot_t pgprot_hyp_device; -extern pgprot_t pgprot_s2; -extern pgprot_t pgprot_s2_device; #define _MOD_PROT(p, b) __pgprot(pgprot_val(p) | (b)) @@ -95,12 +92,6 @@ extern pgprot_t pgprot_s2_device; #define PAGE_READONLY_EXEC _MOD_PROT(pgprot_user, L_PTE_USER | L_PTE_RDONLY) #define PAGE_KERNEL _MOD_PROT(pgprot_kernel, L_PTE_XN) #define PAGE_KERNEL_EXEC pgprot_kernel -#define PAGE_HYP _MOD_PROT(pgprot_kernel, L_PTE_HYP | L_PTE_XN) -#define PAGE_HYP_EXEC _MOD_PROT(pgprot_kernel, L_PTE_HYP | L_PTE_RDONLY) -#define PAGE_HYP_RO _MOD_PROT(pgprot_kernel, L_PTE_HYP | L_PTE_RDONLY | L_PTE_XN) -#define PAGE_HYP_DEVICE _MOD_PROT(pgprot_hyp_device, L_PTE_HYP) -#define PAGE_S2 _MOD_PROT(pgprot_s2, L_PTE_S2_RDONLY | L_PTE_XN) -#define PAGE_S2_DEVICE _MOD_PROT(pgprot_s2_device, L_PTE_S2_RDONLY | L_PTE_XN) #define __PAGE_NONE __pgprot(_L_PTE_DEFAULT | L_PTE_RDONLY | L_PTE_XN | L_PTE_NONE) #define __PAGE_SHARED __pgprot(_L_PTE_DEFAULT | L_PTE_USER | L_PTE_XN) diff --git a/arch/arm/include/asm/sections.h b/arch/arm/include/asm/sections.h index 4ceb4f757d4d..700b8bcdf9bd 100644 --- a/arch/arm/include/asm/sections.h +++ b/arch/arm/include/asm/sections.h @@ -10,8 +10,6 @@ extern char __idmap_text_start[]; extern char __idmap_text_end[]; extern char __entry_text_start[]; extern char __entry_text_end[]; -extern char __hyp_idmap_text_start[]; -extern char __hyp_idmap_text_end[]; static inline bool in_entry_text(unsigned long addr) { @@ -22,9 +20,7 @@ static inline bool in_entry_text(unsigned long addr) static inline bool in_idmap_text(unsigned long addr) { void *a = (void *)addr; - return memory_contains(__idmap_text_start, __idmap_text_end, a, 1) || - memory_contains(__hyp_idmap_text_start, __hyp_idmap_text_end, - a, 1); + return memory_contains(__idmap_text_start, __idmap_text_end, a, 1); } #endif /* _ASM_ARM_SECTIONS_H */ diff --git a/arch/arm/include/asm/stage2_pgtable.h b/arch/arm/include/asm/stage2_pgtable.h deleted file mode 100644 index aaceec7855ec..000000000000 --- a/arch/arm/include/asm/stage2_pgtable.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2016 - ARM Ltd - * - * stage2 page table helpers - */ - -#ifndef __ARM_S2_PGTABLE_H_ -#define __ARM_S2_PGTABLE_H_ - -/* - * kvm_mmu_cache_min_pages() is the number of pages required - * to install a stage-2 translation. We pre-allocate the entry - * level table at VM creation. Since we have a 3 level page-table, - * we need only two pages to add a new mapping. - */ -#define kvm_mmu_cache_min_pages(kvm) 2 - -#define stage2_pgd_none(kvm, pgd) pgd_none(pgd) -#define stage2_pgd_clear(kvm, pgd) pgd_clear(pgd) -#define stage2_pgd_present(kvm, pgd) pgd_present(pgd) -#define stage2_pgd_populate(kvm, pgd, pud) pgd_populate(NULL, pgd, pud) -#define stage2_pud_offset(kvm, pgd, address) pud_offset(pgd, address) -#define stage2_pud_free(kvm, pud) do { } while (0) - -#define stage2_pud_none(kvm, pud) pud_none(pud) -#define stage2_pud_clear(kvm, pud) pud_clear(pud) -#define stage2_pud_present(kvm, pud) pud_present(pud) -#define stage2_pud_populate(kvm, pud, pmd) pud_populate(NULL, pud, pmd) -#define stage2_pmd_offset(kvm, pud, address) pmd_offset(pud, address) -#define stage2_pmd_free(kvm, pmd) free_page((unsigned long)pmd) - -#define stage2_pud_huge(kvm, pud) pud_huge(pud) - -/* Open coded p*d_addr_end that can deal with 64bit addresses */ -static inline phys_addr_t -stage2_pgd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) -{ - phys_addr_t boundary = (addr + PGDIR_SIZE) & PGDIR_MASK; - - return (boundary - 1 < end - 1) ? boundary : end; -} - -#define stage2_pud_addr_end(kvm, addr, end) (end) - -static inline phys_addr_t -stage2_pmd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) -{ - phys_addr_t boundary = (addr + PMD_SIZE) & PMD_MASK; - - return (boundary - 1 < end - 1) ? boundary : end; -} - -#define stage2_pgd_index(kvm, addr) pgd_index(addr) - -#define stage2_pte_table_empty(kvm, ptep) kvm_page_empty(ptep) -#define stage2_pmd_table_empty(kvm, pmdp) kvm_page_empty(pmdp) -#define stage2_pud_table_empty(kvm, pudp) false - -static inline bool kvm_stage2_has_pud(struct kvm *kvm) -{ - return false; -} - -#define S2_PMD_MASK PMD_MASK -#define S2_PMD_SIZE PMD_SIZE -#define S2_PUD_MASK PUD_MASK -#define S2_PUD_SIZE PUD_SIZE - -static inline bool kvm_stage2_has_pmd(struct kvm *kvm) -{ - return true; -} - -#endif /* __ARM_S2_PGTABLE_H_ */ diff --git a/arch/arm/include/asm/virt.h b/arch/arm/include/asm/virt.h index 17c26ccd126d..dd9697b2bde8 100644 --- a/arch/arm/include/asm/virt.h +++ b/arch/arm/include/asm/virt.h @@ -39,8 +39,6 @@ static inline void sync_boot_mode(void) sync_cache_r(&__boot_cpu_mode); } -void __hyp_set_vectors(unsigned long phys_vector_base); -void __hyp_reset_vectors(void); #else #define __boot_cpu_mode (SVC_MODE) #define sync_boot_mode() @@ -67,18 +65,6 @@ static inline bool is_kernel_in_hyp_mode(void) return false; } -static inline bool has_vhe(void) -{ - return false; -} - -/* The section containing the hypervisor idmap text */ -extern char __hyp_idmap_text_start[]; -extern char __hyp_idmap_text_end[]; - -/* The section containing the hypervisor text */ -extern char __hyp_text_start[]; -extern char __hyp_text_end[]; #endif #else @@ -87,9 +73,6 @@ extern char __hyp_text_end[]; #define HVC_SET_VECTORS 0 #define HVC_SOFT_RESTART 1 -#define HVC_RESET_VECTORS 2 - -#define HVC_STUB_HCALL_NR 3 #endif /* __ASSEMBLY__ */ diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h deleted file mode 100644 index 03cd7c19a683..000000000000 --- a/arch/arm/include/uapi/asm/kvm.h +++ /dev/null @@ -1,314 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#ifndef __ARM_KVM_H__ -#define __ARM_KVM_H__ - -#include -#include -#include - -#define __KVM_HAVE_GUEST_DEBUG -#define __KVM_HAVE_IRQ_LINE -#define __KVM_HAVE_READONLY_MEM -#define __KVM_HAVE_VCPU_EVENTS - -#define KVM_COALESCED_MMIO_PAGE_OFFSET 1 - -#define KVM_REG_SIZE(id) \ - (1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT)) - -/* Valid for svc_regs, abt_regs, und_regs, irq_regs in struct kvm_regs */ -#define KVM_ARM_SVC_sp svc_regs[0] -#define KVM_ARM_SVC_lr svc_regs[1] -#define KVM_ARM_SVC_spsr svc_regs[2] -#define KVM_ARM_ABT_sp abt_regs[0] -#define KVM_ARM_ABT_lr abt_regs[1] -#define KVM_ARM_ABT_spsr abt_regs[2] -#define KVM_ARM_UND_sp und_regs[0] -#define KVM_ARM_UND_lr und_regs[1] -#define KVM_ARM_UND_spsr und_regs[2] -#define KVM_ARM_IRQ_sp irq_regs[0] -#define KVM_ARM_IRQ_lr irq_regs[1] -#define KVM_ARM_IRQ_spsr irq_regs[2] - -/* Valid only for fiq_regs in struct kvm_regs */ -#define KVM_ARM_FIQ_r8 fiq_regs[0] -#define KVM_ARM_FIQ_r9 fiq_regs[1] -#define KVM_ARM_FIQ_r10 fiq_regs[2] -#define KVM_ARM_FIQ_fp fiq_regs[3] -#define KVM_ARM_FIQ_ip fiq_regs[4] -#define KVM_ARM_FIQ_sp fiq_regs[5] -#define KVM_ARM_FIQ_lr fiq_regs[6] -#define KVM_ARM_FIQ_spsr fiq_regs[7] - -struct kvm_regs { - struct pt_regs usr_regs; /* R0_usr - R14_usr, PC, CPSR */ - unsigned long svc_regs[3]; /* SP_svc, LR_svc, SPSR_svc */ - unsigned long abt_regs[3]; /* SP_abt, LR_abt, SPSR_abt */ - unsigned long und_regs[3]; /* SP_und, LR_und, SPSR_und */ - unsigned long irq_regs[3]; /* SP_irq, LR_irq, SPSR_irq */ - unsigned long fiq_regs[8]; /* R8_fiq - R14_fiq, SPSR_fiq */ -}; - -/* Supported Processor Types */ -#define KVM_ARM_TARGET_CORTEX_A15 0 -#define KVM_ARM_TARGET_CORTEX_A7 1 -#define KVM_ARM_NUM_TARGETS 2 - -/* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */ -#define KVM_ARM_DEVICE_TYPE_SHIFT 0 -#define KVM_ARM_DEVICE_TYPE_MASK (0xffff << KVM_ARM_DEVICE_TYPE_SHIFT) -#define KVM_ARM_DEVICE_ID_SHIFT 16 -#define KVM_ARM_DEVICE_ID_MASK (0xffff << KVM_ARM_DEVICE_ID_SHIFT) - -/* Supported device IDs */ -#define KVM_ARM_DEVICE_VGIC_V2 0 - -/* Supported VGIC address types */ -#define KVM_VGIC_V2_ADDR_TYPE_DIST 0 -#define KVM_VGIC_V2_ADDR_TYPE_CPU 1 - -#define KVM_VGIC_V2_DIST_SIZE 0x1000 -#define KVM_VGIC_V2_CPU_SIZE 0x2000 - -/* Supported VGICv3 address types */ -#define KVM_VGIC_V3_ADDR_TYPE_DIST 2 -#define KVM_VGIC_V3_ADDR_TYPE_REDIST 3 -#define KVM_VGIC_ITS_ADDR_TYPE 4 -#define KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION 5 - -#define KVM_VGIC_V3_DIST_SIZE SZ_64K -#define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) -#define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K) - -#define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ -#define KVM_ARM_VCPU_PSCI_0_2 1 /* CPU uses PSCI v0.2 */ - -struct kvm_vcpu_init { - __u32 target; - __u32 features[7]; -}; - -struct kvm_sregs { -}; - -struct kvm_fpu { -}; - -struct kvm_guest_debug_arch { -}; - -struct kvm_debug_exit_arch { -}; - -struct kvm_sync_regs { - /* Used with KVM_CAP_ARM_USER_IRQ */ - __u64 device_irq_level; -}; - -struct kvm_arch_memory_slot { -}; - -/* for KVM_GET/SET_VCPU_EVENTS */ -struct kvm_vcpu_events { - struct { - __u8 serror_pending; - __u8 serror_has_esr; - __u8 ext_dabt_pending; - /* Align it to 8 bytes */ - __u8 pad[5]; - __u64 serror_esr; - } exception; - __u32 reserved[12]; -}; - -/* If you need to interpret the index values, here is the key: */ -#define KVM_REG_ARM_COPROC_MASK 0x000000000FFF0000 -#define KVM_REG_ARM_COPROC_SHIFT 16 -#define KVM_REG_ARM_32_OPC2_MASK 0x0000000000000007 -#define KVM_REG_ARM_32_OPC2_SHIFT 0 -#define KVM_REG_ARM_OPC1_MASK 0x0000000000000078 -#define KVM_REG_ARM_OPC1_SHIFT 3 -#define KVM_REG_ARM_CRM_MASK 0x0000000000000780 -#define KVM_REG_ARM_CRM_SHIFT 7 -#define KVM_REG_ARM_32_CRN_MASK 0x0000000000007800 -#define KVM_REG_ARM_32_CRN_SHIFT 11 -/* - * For KVM currently all guest registers are nonsecure, but we reserve a bit - * in the encoding to distinguish secure from nonsecure for AArch32 system - * registers that are banked by security. This is 1 for the secure banked - * register, and 0 for the nonsecure banked register or if the register is - * not banked by security. - */ -#define KVM_REG_ARM_SECURE_MASK 0x0000000010000000 -#define KVM_REG_ARM_SECURE_SHIFT 28 - -#define ARM_CP15_REG_SHIFT_MASK(x,n) \ - (((x) << KVM_REG_ARM_ ## n ## _SHIFT) & KVM_REG_ARM_ ## n ## _MASK) - -#define __ARM_CP15_REG(op1,crn,crm,op2) \ - (KVM_REG_ARM | (15 << KVM_REG_ARM_COPROC_SHIFT) | \ - ARM_CP15_REG_SHIFT_MASK(op1, OPC1) | \ - ARM_CP15_REG_SHIFT_MASK(crn, 32_CRN) | \ - ARM_CP15_REG_SHIFT_MASK(crm, CRM) | \ - ARM_CP15_REG_SHIFT_MASK(op2, 32_OPC2)) - -#define ARM_CP15_REG32(...) (__ARM_CP15_REG(__VA_ARGS__) | KVM_REG_SIZE_U32) - -#define __ARM_CP15_REG64(op1,crm) \ - (__ARM_CP15_REG(op1, 0, crm, 0) | KVM_REG_SIZE_U64) -#define ARM_CP15_REG64(...) __ARM_CP15_REG64(__VA_ARGS__) - -/* PL1 Physical Timer Registers */ -#define KVM_REG_ARM_PTIMER_CTL ARM_CP15_REG32(0, 14, 2, 1) -#define KVM_REG_ARM_PTIMER_CNT ARM_CP15_REG64(0, 14) -#define KVM_REG_ARM_PTIMER_CVAL ARM_CP15_REG64(2, 14) - -/* Virtual Timer Registers */ -#define KVM_REG_ARM_TIMER_CTL ARM_CP15_REG32(0, 14, 3, 1) -#define KVM_REG_ARM_TIMER_CNT ARM_CP15_REG64(1, 14) -#define KVM_REG_ARM_TIMER_CVAL ARM_CP15_REG64(3, 14) - -/* Normal registers are mapped as coprocessor 16. */ -#define KVM_REG_ARM_CORE (0x0010 << KVM_REG_ARM_COPROC_SHIFT) -#define KVM_REG_ARM_CORE_REG(name) (offsetof(struct kvm_regs, name) / 4) - -/* Some registers need more space to represent values. */ -#define KVM_REG_ARM_DEMUX (0x0011 << KVM_REG_ARM_COPROC_SHIFT) -#define KVM_REG_ARM_DEMUX_ID_MASK 0x000000000000FF00 -#define KVM_REG_ARM_DEMUX_ID_SHIFT 8 -#define KVM_REG_ARM_DEMUX_ID_CCSIDR (0x00 << KVM_REG_ARM_DEMUX_ID_SHIFT) -#define KVM_REG_ARM_DEMUX_VAL_MASK 0x00000000000000FF -#define KVM_REG_ARM_DEMUX_VAL_SHIFT 0 - -/* VFP registers: we could overload CP10 like ARM does, but that's ugly. */ -#define KVM_REG_ARM_VFP (0x0012 << KVM_REG_ARM_COPROC_SHIFT) -#define KVM_REG_ARM_VFP_MASK 0x000000000000FFFF -#define KVM_REG_ARM_VFP_BASE_REG 0x0 -#define KVM_REG_ARM_VFP_FPSID 0x1000 -#define KVM_REG_ARM_VFP_FPSCR 0x1001 -#define KVM_REG_ARM_VFP_MVFR1 0x1006 -#define KVM_REG_ARM_VFP_MVFR0 0x1007 -#define KVM_REG_ARM_VFP_FPEXC 0x1008 -#define KVM_REG_ARM_VFP_FPINST 0x1009 -#define KVM_REG_ARM_VFP_FPINST2 0x100A - -/* KVM-as-firmware specific pseudo-registers */ -#define KVM_REG_ARM_FW (0x0014 << KVM_REG_ARM_COPROC_SHIFT) -#define KVM_REG_ARM_FW_REG(r) (KVM_REG_ARM | KVM_REG_SIZE_U64 | \ - KVM_REG_ARM_FW | ((r) & 0xffff)) -#define KVM_REG_ARM_PSCI_VERSION KVM_REG_ARM_FW_REG(0) -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1 KVM_REG_ARM_FW_REG(1) - /* Higher values mean better protection. */ -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL 0 -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_AVAIL 1 -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_REQUIRED 2 -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2 KVM_REG_ARM_FW_REG(2) - /* Higher values mean better protection. */ -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL 0 -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_UNKNOWN 1 -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL 2 -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED 3 -#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED (1U << 4) - -/* Device Control API: ARM VGIC */ -#define KVM_DEV_ARM_VGIC_GRP_ADDR 0 -#define KVM_DEV_ARM_VGIC_GRP_DIST_REGS 1 -#define KVM_DEV_ARM_VGIC_GRP_CPU_REGS 2 -#define KVM_DEV_ARM_VGIC_CPUID_SHIFT 32 -#define KVM_DEV_ARM_VGIC_CPUID_MASK (0xffULL << KVM_DEV_ARM_VGIC_CPUID_SHIFT) -#define KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT 32 -#define KVM_DEV_ARM_VGIC_V3_MPIDR_MASK \ - (0xffffffffULL << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT) -#define KVM_DEV_ARM_VGIC_OFFSET_SHIFT 0 -#define KVM_DEV_ARM_VGIC_OFFSET_MASK (0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT) -#define KVM_DEV_ARM_VGIC_SYSREG_INSTR_MASK (0xffff) -#define KVM_DEV_ARM_VGIC_GRP_NR_IRQS 3 -#define KVM_DEV_ARM_VGIC_GRP_CTRL 4 -#define KVM_DEV_ARM_VGIC_GRP_REDIST_REGS 5 -#define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6 -#define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7 -#define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 -#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 -#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ - (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT) -#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INTID_MASK 0x3ff -#define VGIC_LEVEL_INFO_LINE_LEVEL 0 - -/* Device Control API on vcpu fd */ -#define KVM_ARM_VCPU_PMU_V3_CTRL 0 -#define KVM_ARM_VCPU_PMU_V3_IRQ 0 -#define KVM_ARM_VCPU_PMU_V3_INIT 1 -#define KVM_ARM_VCPU_TIMER_CTRL 1 -#define KVM_ARM_VCPU_TIMER_IRQ_VTIMER 0 -#define KVM_ARM_VCPU_TIMER_IRQ_PTIMER 1 - -#define KVM_DEV_ARM_VGIC_CTRL_INIT 0 -#define KVM_DEV_ARM_ITS_SAVE_TABLES 1 -#define KVM_DEV_ARM_ITS_RESTORE_TABLES 2 -#define KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES 3 -#define KVM_DEV_ARM_ITS_CTRL_RESET 4 - -/* KVM_IRQ_LINE irq field index values */ -#define KVM_ARM_IRQ_VCPU2_SHIFT 28 -#define KVM_ARM_IRQ_VCPU2_MASK 0xf -#define KVM_ARM_IRQ_TYPE_SHIFT 24 -#define KVM_ARM_IRQ_TYPE_MASK 0xf -#define KVM_ARM_IRQ_VCPU_SHIFT 16 -#define KVM_ARM_IRQ_VCPU_MASK 0xff -#define KVM_ARM_IRQ_NUM_SHIFT 0 -#define KVM_ARM_IRQ_NUM_MASK 0xffff - -/* irq_type field */ -#define KVM_ARM_IRQ_TYPE_CPU 0 -#define KVM_ARM_IRQ_TYPE_SPI 1 -#define KVM_ARM_IRQ_TYPE_PPI 2 - -/* out-of-kernel GIC cpu interrupt injection irq_number field */ -#define KVM_ARM_IRQ_CPU_IRQ 0 -#define KVM_ARM_IRQ_CPU_FIQ 1 - -/* - * This used to hold the highest supported SPI, but it is now obsolete - * and only here to provide source code level compatibility with older - * userland. The highest SPI number can be set via KVM_DEV_ARM_VGIC_GRP_NR_IRQS. - */ -#ifndef __KERNEL__ -#define KVM_ARM_IRQ_GIC_MAX 127 -#endif - -/* One single KVM irqchip, ie. the VGIC */ -#define KVM_NR_IRQCHIPS 1 - -/* PSCI interface */ -#define KVM_PSCI_FN_BASE 0x95c1ba5e -#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n)) - -#define KVM_PSCI_FN_CPU_SUSPEND KVM_PSCI_FN(0) -#define KVM_PSCI_FN_CPU_OFF KVM_PSCI_FN(1) -#define KVM_PSCI_FN_CPU_ON KVM_PSCI_FN(2) -#define KVM_PSCI_FN_MIGRATE KVM_PSCI_FN(3) - -#define KVM_PSCI_RET_SUCCESS PSCI_RET_SUCCESS -#define KVM_PSCI_RET_NI PSCI_RET_NOT_SUPPORTED -#define KVM_PSCI_RET_INVAL PSCI_RET_INVALID_PARAMS -#define KVM_PSCI_RET_DENIED PSCI_RET_DENIED - -#endif /* __ARM_KVM_H__ */ diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index c773b829ee8e..c036a4a2f8e2 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -11,9 +11,6 @@ #include #include #include -#ifdef CONFIG_KVM_ARM_HOST -#include -#endif #include #include #include @@ -167,14 +164,6 @@ int main(void) DEFINE(CACHE_WRITEBACK_ORDER, __CACHE_WRITEBACK_ORDER); DEFINE(CACHE_WRITEBACK_GRANULE, __CACHE_WRITEBACK_GRANULE); BLANK(); -#ifdef CONFIG_KVM_ARM_HOST - DEFINE(VCPU_GUEST_CTXT, offsetof(struct kvm_vcpu, arch.ctxt)); - DEFINE(VCPU_HOST_CTXT, offsetof(struct kvm_vcpu, arch.host_cpu_context)); - DEFINE(CPU_CTXT_VFP, offsetof(struct kvm_cpu_context, vfp)); - DEFINE(CPU_CTXT_GP_REGS, offsetof(struct kvm_cpu_context, gp_regs)); - DEFINE(GP_REGS_USR, offsetof(struct kvm_regs, usr_regs)); -#endif - BLANK(); #ifdef CONFIG_VDSO DEFINE(VDSO_DATA_SIZE, sizeof(union vdso_data_store)); #endif diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S index 6607fa817bba..26d8e03b1dd3 100644 --- a/arch/arm/kernel/hyp-stub.S +++ b/arch/arm/kernel/hyp-stub.S @@ -189,19 +189,19 @@ ARM_BE8(orr r7, r7, #(1 << 25)) @ HSCTLR.EE ENDPROC(__hyp_stub_install_secondary) __hyp_stub_do_trap: +#ifdef ZIMAGE teq r0, #HVC_SET_VECTORS bne 1f + /* Only the ZIMAGE stubs can change the HYP vectors */ mcr p15, 4, r1, c12, c0, 0 @ set HVBAR b __hyp_stub_exit +#endif 1: teq r0, #HVC_SOFT_RESTART - bne 1f + bne 2f bx r1 -1: teq r0, #HVC_RESET_VECTORS - beq __hyp_stub_exit - - ldr r0, =HVC_STUB_ERR +2: ldr r0, =HVC_STUB_ERR __ERET __hyp_stub_exit: @@ -210,26 +210,9 @@ __hyp_stub_exit: ENDPROC(__hyp_stub_do_trap) /* - * __hyp_set_vectors: Call this after boot to set the initial hypervisor - * vectors as part of hypervisor installation. On an SMP system, this should - * be called on each CPU. - * - * r0 must be the physical address of the new vector table (which must lie in - * the bottom 4GB of physical address space. - * - * r0 must be 32-byte aligned. - * - * Before calling this, you must check that the stub hypervisor is installed - * everywhere, by waiting for any secondary CPUs to be brought up and then - * checking that BOOT_CPU_MODE_HAVE_HYP(__boot_cpu_mode) is true. - * - * If not, there is a pre-existing hypervisor, some CPUs failed to boot, or - * something else went wrong... in such cases, trying to install a new - * hypervisor is unlikely to work as desired. - * - * When you call into your shiny new hypervisor, sp_hyp will contain junk, - * so you will need to set that to something sensible at the new hypervisor's - * initialisation entry point. + * __hyp_set_vectors is only used when ZIMAGE must bounce between HYP + * and SVC. For the kernel itself, the vectors are set once and for + * all by the stubs. */ ENTRY(__hyp_set_vectors) mov r1, r0 @@ -245,12 +228,6 @@ ENTRY(__hyp_soft_restart) ret lr ENDPROC(__hyp_soft_restart) -ENTRY(__hyp_reset_vectors) - mov r0, #HVC_RESET_VECTORS - __HVC(0) - ret lr -ENDPROC(__hyp_reset_vectors) - #ifndef ZIMAGE .align 2 .L__boot_cpu_mode_offset: diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S index 21b8b271c80d..6d2be994ae58 100644 --- a/arch/arm/kernel/vmlinux-xip.lds.S +++ b/arch/arm/kernel/vmlinux-xip.lds.S @@ -162,14 +162,6 @@ SECTIONS ASSERT((__proc_info_end - __proc_info_begin), "missing CPU support") ASSERT((__arch_info_end - __arch_info_begin), "no machine record defined") -/* - * The HYP init code can't be more than a page long, - * and should not cross a page boundary. - * The above comment applies as well. - */ -ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & PAGE_MASK) <= PAGE_SIZE, - "HYP init code too big or misaligned") - #ifdef CONFIG_XIP_DEFLATED_DATA /* * The .bss is used as a stack area for __inflate_kernel_data() whose stack diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index 319ccb10846a..88a720da443b 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -170,12 +170,4 @@ __start_rodata_section_aligned = ALIGN(__start_rodata, 1 << SECTION_SHIFT); ASSERT((__proc_info_end - __proc_info_begin), "missing CPU support") ASSERT((__arch_info_end - __arch_info_begin), "no machine record defined") -/* - * The HYP init code can't be more than a page long, - * and should not cross a page boundary. - * The above comment applies as well. - */ -ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & PAGE_MASK) <= PAGE_SIZE, - "HYP init code too big or misaligned") - #endif /* CONFIG_XIP_KERNEL */ diff --git a/arch/arm/kernel/vmlinux.lds.h b/arch/arm/kernel/vmlinux.lds.h index 8247bc15addc..381a8e105fa5 100644 --- a/arch/arm/kernel/vmlinux.lds.h +++ b/arch/arm/kernel/vmlinux.lds.h @@ -31,20 +31,11 @@ *(.proc.info.init) \ __proc_info_end = .; -#define HYPERVISOR_TEXT \ - __hyp_text_start = .; \ - *(.hyp.text) \ - __hyp_text_end = .; - #define IDMAP_TEXT \ ALIGN_FUNCTION(); \ __idmap_text_start = .; \ *(.idmap.text) \ __idmap_text_end = .; \ - . = ALIGN(PAGE_SIZE); \ - __hyp_idmap_text_start = .; \ - *(.hyp.idmap.text) \ - __hyp_idmap_text_end = .; #define ARM_DISCARD \ *(.ARM.exidx.exit.text) \ @@ -72,7 +63,6 @@ SCHED_TEXT \ CPUIDLE_TEXT \ LOCK_TEXT \ - HYPERVISOR_TEXT \ KPROBES_TEXT \ *(.gnu.warning) \ *(.glue_7) \ diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig deleted file mode 100644 index f591026347a5..000000000000 --- a/arch/arm/kvm/Kconfig +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# KVM configuration -# - -source "virt/kvm/Kconfig" -source "virt/lib/Kconfig" - -menuconfig VIRTUALIZATION - bool "Virtualization" - ---help--- - Say Y here to get to see options for using your Linux host to run - other operating systems inside virtual machines (guests). - This option alone does not add any kernel code. - - If you say N, all options in this submenu will be skipped and - disabled. - -if VIRTUALIZATION - -config KVM - bool "Kernel-based Virtual Machine (KVM) support" - depends on MMU && OF - select PREEMPT_NOTIFIERS - select ARM_GIC - select ARM_GIC_V3 - select ARM_GIC_V3_ITS - select HAVE_KVM_CPU_RELAX_INTERCEPT - select HAVE_KVM_ARCH_TLB_FLUSH_ALL - select KVM_MMIO - select KVM_ARM_HOST - select KVM_GENERIC_DIRTYLOG_READ_PROTECT - select SRCU - select MMU_NOTIFIER - select KVM_VFIO - select HAVE_KVM_EVENTFD - select HAVE_KVM_IRQFD - select HAVE_KVM_IRQCHIP - select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_MSI - select IRQ_BYPASS_MANAGER - select HAVE_KVM_IRQ_BYPASS - depends on ARM_VIRT_EXT && ARM_LPAE && ARM_ARCH_TIMER - ---help--- - Support hosting virtualized guest machines. - - This module provides access to the hardware capabilities through - a character device node named /dev/kvm. - - If unsure, say N. - -config KVM_ARM_HOST - bool - ---help--- - Provides host support for ARM processors. - -source "drivers/vhost/Kconfig" - -endif # VIRTUALIZATION diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile deleted file mode 100644 index e442d82821df..000000000000 --- a/arch/arm/kvm/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for Kernel-based Virtual Machine module -# - -plus_virt := $(call as-instr,.arch_extension virt,+virt) -ifeq ($(plus_virt),+virt) - plus_virt_def := -DREQUIRES_VIRT=1 -endif - -KVM := ../../../virt/kvm - -ccflags-y += -I $(srctree)/$(src) -I $(srctree)/virt/kvm/arm/vgic -CFLAGS_$(KVM)/arm/arm.o := $(plus_virt_def) - -AFLAGS_init.o := -Wa,-march=armv7-a$(plus_virt) -AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt) - -kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o $(KVM)/vfio.o - -obj-$(CONFIG_KVM_ARM_HOST) += hyp/ - -obj-y += kvm-arm.o init.o interrupts.o -obj-y += handle_exit.o guest.o emulate.o reset.o -obj-y += coproc.o coproc_a15.o coproc_a7.o vgic-v3-coproc.o -obj-y += $(KVM)/arm/arm.o $(KVM)/arm/mmu.o $(KVM)/arm/mmio.o -obj-y += $(KVM)/arm/psci.o $(KVM)/arm/perf.o $(KVM)/arm/hypercalls.o -obj-y += $(KVM)/arm/aarch32.o - -obj-y += $(KVM)/arm/vgic/vgic.o -obj-y += $(KVM)/arm/vgic/vgic-init.o -obj-y += $(KVM)/arm/vgic/vgic-irqfd.o -obj-y += $(KVM)/arm/vgic/vgic-v2.o -obj-y += $(KVM)/arm/vgic/vgic-v3.o -obj-y += $(KVM)/arm/vgic/vgic-v4.o -obj-y += $(KVM)/arm/vgic/vgic-mmio.o -obj-y += $(KVM)/arm/vgic/vgic-mmio-v2.o -obj-y += $(KVM)/arm/vgic/vgic-mmio-v3.o -obj-y += $(KVM)/arm/vgic/vgic-kvm-device.o -obj-y += $(KVM)/arm/vgic/vgic-its.o -obj-y += $(KVM)/arm/vgic/vgic-debug.o -obj-y += $(KVM)/irqchip.o -obj-y += $(KVM)/arm/arch_timer.o diff --git a/arch/arm/kvm/coproc.c b/arch/arm/kvm/coproc.c deleted file mode 100644 index 07745ee022a1..000000000000 --- a/arch/arm/kvm/coproc.c +++ /dev/null @@ -1,1455 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Authors: Rusty Russell - * Christoffer Dall - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../vfp/vfpinstr.h" - -#define CREATE_TRACE_POINTS -#include "trace.h" -#include "coproc.h" - - -/****************************************************************************** - * Co-processor emulation - *****************************************************************************/ - -static bool write_to_read_only(struct kvm_vcpu *vcpu, - const struct coproc_params *params) -{ - WARN_ONCE(1, "CP15 write to read-only register\n"); - print_cp_instr(params); - kvm_inject_undefined(vcpu); - return false; -} - -static bool read_from_write_only(struct kvm_vcpu *vcpu, - const struct coproc_params *params) -{ - WARN_ONCE(1, "CP15 read to write-only register\n"); - print_cp_instr(params); - kvm_inject_undefined(vcpu); - return false; -} - -/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */ -static u32 cache_levels; - -/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */ -#define CSSELR_MAX 12 - -/* - * kvm_vcpu_arch.cp15 holds cp15 registers as an array of u32, but some - * of cp15 registers can be viewed either as couple of two u32 registers - * or one u64 register. Current u64 register encoding is that least - * significant u32 word is followed by most significant u32 word. - */ -static inline void vcpu_cp15_reg64_set(struct kvm_vcpu *vcpu, - const struct coproc_reg *r, - u64 val) -{ - vcpu_cp15(vcpu, r->reg) = val & 0xffffffff; - vcpu_cp15(vcpu, r->reg + 1) = val >> 32; -} - -static inline u64 vcpu_cp15_reg64_get(struct kvm_vcpu *vcpu, - const struct coproc_reg *r) -{ - u64 val; - - val = vcpu_cp15(vcpu, r->reg + 1); - val = val << 32; - val = val | vcpu_cp15(vcpu, r->reg); - return val; -} - -int kvm_handle_cp10_id(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - kvm_inject_undefined(vcpu); - return 1; -} - -int kvm_handle_cp_0_13_access(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - /* - * We can get here, if the host has been built without VFPv3 support, - * but the guest attempted a floating point operation. - */ - kvm_inject_undefined(vcpu); - return 1; -} - -int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - kvm_inject_undefined(vcpu); - return 1; -} - -static void reset_mpidr(struct kvm_vcpu *vcpu, const struct coproc_reg *r) -{ - /* - * Compute guest MPIDR. We build a virtual cluster out of the - * vcpu_id, but we read the 'U' bit from the underlying - * hardware directly. - */ - vcpu_cp15(vcpu, c0_MPIDR) = ((read_cpuid_mpidr() & MPIDR_SMP_BITMASK) | - ((vcpu->vcpu_id >> 2) << MPIDR_LEVEL_BITS) | - (vcpu->vcpu_id & 3)); -} - -/* TRM entries A7:4.3.31 A15:4.3.28 - RO WI */ -static bool access_actlr(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - if (p->is_write) - return ignore_write(vcpu, p); - - *vcpu_reg(vcpu, p->Rt1) = vcpu_cp15(vcpu, c1_ACTLR); - return true; -} - -/* TRM entries A7:4.3.56, A15:4.3.60 - R/O. */ -static bool access_cbar(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - if (p->is_write) - return write_to_read_only(vcpu, p); - return read_zero(vcpu, p); -} - -/* TRM entries A7:4.3.49, A15:4.3.48 - R/O WI */ -static bool access_l2ctlr(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - if (p->is_write) - return ignore_write(vcpu, p); - - *vcpu_reg(vcpu, p->Rt1) = vcpu_cp15(vcpu, c9_L2CTLR); - return true; -} - -static void reset_l2ctlr(struct kvm_vcpu *vcpu, const struct coproc_reg *r) -{ - u32 l2ctlr, ncores; - - asm volatile("mrc p15, 1, %0, c9, c0, 2\n" : "=r" (l2ctlr)); - l2ctlr &= ~(3 << 24); - ncores = atomic_read(&vcpu->kvm->online_vcpus) - 1; - /* How many cores in the current cluster and the next ones */ - ncores -= (vcpu->vcpu_id & ~3); - /* Cap it to the maximum number of cores in a single cluster */ - ncores = min(ncores, 3U); - l2ctlr |= (ncores & 3) << 24; - - vcpu_cp15(vcpu, c9_L2CTLR) = l2ctlr; -} - -static void reset_actlr(struct kvm_vcpu *vcpu, const struct coproc_reg *r) -{ - u32 actlr; - - /* ACTLR contains SMP bit: make sure you create all cpus first! */ - asm volatile("mrc p15, 0, %0, c1, c0, 1\n" : "=r" (actlr)); - /* Make the SMP bit consistent with the guest configuration */ - if (atomic_read(&vcpu->kvm->online_vcpus) > 1) - actlr |= 1U << 6; - else - actlr &= ~(1U << 6); - - vcpu_cp15(vcpu, c1_ACTLR) = actlr; -} - -/* - * TRM entries: A7:4.3.50, A15:4.3.49 - * R/O WI (even if NSACR.NS_L2ERR, a write of 1 is ignored). - */ -static bool access_l2ectlr(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - if (p->is_write) - return ignore_write(vcpu, p); - - *vcpu_reg(vcpu, p->Rt1) = 0; - return true; -} - -/* - * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized). - */ -static bool access_dcsw(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - if (!p->is_write) - return read_from_write_only(vcpu, p); - - kvm_set_way_flush(vcpu); - return true; -} - -/* - * Generic accessor for VM registers. Only called as long as HCR_TVM - * is set. If the guest enables the MMU, we stop trapping the VM - * sys_regs and leave it in complete control of the caches. - * - * Used by the cpu-specific code. - */ -bool access_vm_reg(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - bool was_enabled = vcpu_has_cache_enabled(vcpu); - - BUG_ON(!p->is_write); - - vcpu_cp15(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt1); - if (p->is_64bit) - vcpu_cp15(vcpu, r->reg + 1) = *vcpu_reg(vcpu, p->Rt2); - - kvm_toggle_cache(vcpu, was_enabled); - return true; -} - -static bool access_gic_sgi(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - u64 reg; - bool g1; - - if (!p->is_write) - return read_from_write_only(vcpu, p); - - reg = (u64)*vcpu_reg(vcpu, p->Rt2) << 32; - reg |= *vcpu_reg(vcpu, p->Rt1) ; - - /* - * In a system where GICD_CTLR.DS=1, a ICC_SGI0R access generates - * Group0 SGIs only, while ICC_SGI1R can generate either group, - * depending on the SGI configuration. ICC_ASGI1R is effectively - * equivalent to ICC_SGI0R, as there is no "alternative" secure - * group. - */ - switch (p->Op1) { - default: /* Keep GCC quiet */ - case 0: /* ICC_SGI1R */ - g1 = true; - break; - case 1: /* ICC_ASGI1R */ - case 2: /* ICC_SGI0R */ - g1 = false; - break; - } - - vgic_v3_dispatch_sgi(vcpu, reg, g1); - - return true; -} - -static bool access_gic_sre(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - if (p->is_write) - return ignore_write(vcpu, p); - - *vcpu_reg(vcpu, p->Rt1) = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre; - - return true; -} - -static bool access_cntp_tval(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - u32 val; - - if (p->is_write) { - val = *vcpu_reg(vcpu, p->Rt1); - kvm_arm_timer_write_sysreg(vcpu, - TIMER_PTIMER, TIMER_REG_TVAL, val); - } else { - val = kvm_arm_timer_read_sysreg(vcpu, - TIMER_PTIMER, TIMER_REG_TVAL); - *vcpu_reg(vcpu, p->Rt1) = val; - } - - return true; -} - -static bool access_cntp_ctl(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - u32 val; - - if (p->is_write) { - val = *vcpu_reg(vcpu, p->Rt1); - kvm_arm_timer_write_sysreg(vcpu, - TIMER_PTIMER, TIMER_REG_CTL, val); - } else { - val = kvm_arm_timer_read_sysreg(vcpu, - TIMER_PTIMER, TIMER_REG_CTL); - *vcpu_reg(vcpu, p->Rt1) = val; - } - - return true; -} - -static bool access_cntp_cval(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - u64 val; - - if (p->is_write) { - val = (u64)*vcpu_reg(vcpu, p->Rt2) << 32; - val |= *vcpu_reg(vcpu, p->Rt1); - kvm_arm_timer_write_sysreg(vcpu, - TIMER_PTIMER, TIMER_REG_CVAL, val); - } else { - val = kvm_arm_timer_read_sysreg(vcpu, - TIMER_PTIMER, TIMER_REG_CVAL); - *vcpu_reg(vcpu, p->Rt1) = val; - *vcpu_reg(vcpu, p->Rt2) = val >> 32; - } - - return true; -} - -/* - * We could trap ID_DFR0 and tell the guest we don't support performance - * monitoring. Unfortunately the patch to make the kernel check ID_DFR0 was - * NAKed, so it will read the PMCR anyway. - * - * Therefore we tell the guest we have 0 counters. Unfortunately, we - * must always support PMCCNTR (the cycle counter): we just RAZ/WI for - * all PM registers, which doesn't crash the guest kernel at least. - */ -static bool trap_raz_wi(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r) -{ - if (p->is_write) - return ignore_write(vcpu, p); - else - return read_zero(vcpu, p); -} - -#define access_pmcr trap_raz_wi -#define access_pmcntenset trap_raz_wi -#define access_pmcntenclr trap_raz_wi -#define access_pmovsr trap_raz_wi -#define access_pmselr trap_raz_wi -#define access_pmceid0 trap_raz_wi -#define access_pmceid1 trap_raz_wi -#define access_pmccntr trap_raz_wi -#define access_pmxevtyper trap_raz_wi -#define access_pmxevcntr trap_raz_wi -#define access_pmuserenr trap_raz_wi -#define access_pmintenset trap_raz_wi -#define access_pmintenclr trap_raz_wi - -/* Architected CP15 registers. - * CRn denotes the primary register number, but is copied to the CRm in the - * user space API for 64-bit register access in line with the terminology used - * in the ARM ARM. - * Important: Must be sorted ascending by CRn, CRM, Op1, Op2 and with 64-bit - * registers preceding 32-bit ones. - */ -static const struct coproc_reg cp15_regs[] = { - /* MPIDR: we use VMPIDR for guest access. */ - { CRn( 0), CRm( 0), Op1( 0), Op2( 5), is32, - NULL, reset_mpidr, c0_MPIDR }, - - /* CSSELR: swapped by interrupt.S. */ - { CRn( 0), CRm( 0), Op1( 2), Op2( 0), is32, - NULL, reset_unknown, c0_CSSELR }, - - /* ACTLR: trapped by HCR.TAC bit. */ - { CRn( 1), CRm( 0), Op1( 0), Op2( 1), is32, - access_actlr, reset_actlr, c1_ACTLR }, - - /* CPACR: swapped by interrupt.S. */ - { CRn( 1), CRm( 0), Op1( 0), Op2( 2), is32, - NULL, reset_val, c1_CPACR, 0x00000000 }, - - /* TTBR0/TTBR1/TTBCR: swapped by interrupt.S. */ - { CRm64( 2), Op1( 0), is64, access_vm_reg, reset_unknown64, c2_TTBR0 }, - { CRn(2), CRm( 0), Op1( 0), Op2( 0), is32, - access_vm_reg, reset_unknown, c2_TTBR0 }, - { CRn(2), CRm( 0), Op1( 0), Op2( 1), is32, - access_vm_reg, reset_unknown, c2_TTBR1 }, - { CRn( 2), CRm( 0), Op1( 0), Op2( 2), is32, - access_vm_reg, reset_val, c2_TTBCR, 0x00000000 }, - { CRm64( 2), Op1( 1), is64, access_vm_reg, reset_unknown64, c2_TTBR1 }, - - - /* DACR: swapped by interrupt.S. */ - { CRn( 3), CRm( 0), Op1( 0), Op2( 0), is32, - access_vm_reg, reset_unknown, c3_DACR }, - - /* DFSR/IFSR/ADFSR/AIFSR: swapped by interrupt.S. */ - { CRn( 5), CRm( 0), Op1( 0), Op2( 0), is32, - access_vm_reg, reset_unknown, c5_DFSR }, - { CRn( 5), CRm( 0), Op1( 0), Op2( 1), is32, - access_vm_reg, reset_unknown, c5_IFSR }, - { CRn( 5), CRm( 1), Op1( 0), Op2( 0), is32, - access_vm_reg, reset_unknown, c5_ADFSR }, - { CRn( 5), CRm( 1), Op1( 0), Op2( 1), is32, - access_vm_reg, reset_unknown, c5_AIFSR }, - - /* DFAR/IFAR: swapped by interrupt.S. */ - { CRn( 6), CRm( 0), Op1( 0), Op2( 0), is32, - access_vm_reg, reset_unknown, c6_DFAR }, - { CRn( 6), CRm( 0), Op1( 0), Op2( 2), is32, - access_vm_reg, reset_unknown, c6_IFAR }, - - /* PAR swapped by interrupt.S */ - { CRm64( 7), Op1( 0), is64, NULL, reset_unknown64, c7_PAR }, - - /* - * DC{C,I,CI}SW operations: - */ - { CRn( 7), CRm( 6), Op1( 0), Op2( 2), is32, access_dcsw}, - { CRn( 7), CRm(10), Op1( 0), Op2( 2), is32, access_dcsw}, - { CRn( 7), CRm(14), Op1( 0), Op2( 2), is32, access_dcsw}, - /* - * L2CTLR access (guest wants to know #CPUs). - */ - { CRn( 9), CRm( 0), Op1( 1), Op2( 2), is32, - access_l2ctlr, reset_l2ctlr, c9_L2CTLR }, - { CRn( 9), CRm( 0), Op1( 1), Op2( 3), is32, access_l2ectlr}, - - /* - * Dummy performance monitor implementation. - */ - { CRn( 9), CRm(12), Op1( 0), Op2( 0), is32, access_pmcr}, - { CRn( 9), CRm(12), Op1( 0), Op2( 1), is32, access_pmcntenset}, - { CRn( 9), CRm(12), Op1( 0), Op2( 2), is32, access_pmcntenclr}, - { CRn( 9), CRm(12), Op1( 0), Op2( 3), is32, access_pmovsr}, - { CRn( 9), CRm(12), Op1( 0), Op2( 5), is32, access_pmselr}, - { CRn( 9), CRm(12), Op1( 0), Op2( 6), is32, access_pmceid0}, - { CRn( 9), CRm(12), Op1( 0), Op2( 7), is32, access_pmceid1}, - { CRn( 9), CRm(13), Op1( 0), Op2( 0), is32, access_pmccntr}, - { CRn( 9), CRm(13), Op1( 0), Op2( 1), is32, access_pmxevtyper}, - { CRn( 9), CRm(13), Op1( 0), Op2( 2), is32, access_pmxevcntr}, - { CRn( 9), CRm(14), Op1( 0), Op2( 0), is32, access_pmuserenr}, - { CRn( 9), CRm(14), Op1( 0), Op2( 1), is32, access_pmintenset}, - { CRn( 9), CRm(14), Op1( 0), Op2( 2), is32, access_pmintenclr}, - - /* PRRR/NMRR (aka MAIR0/MAIR1): swapped by interrupt.S. */ - { CRn(10), CRm( 2), Op1( 0), Op2( 0), is32, - access_vm_reg, reset_unknown, c10_PRRR}, - { CRn(10), CRm( 2), Op1( 0), Op2( 1), is32, - access_vm_reg, reset_unknown, c10_NMRR}, - - /* AMAIR0/AMAIR1: swapped by interrupt.S. */ - { CRn(10), CRm( 3), Op1( 0), Op2( 0), is32, - access_vm_reg, reset_unknown, c10_AMAIR0}, - { CRn(10), CRm( 3), Op1( 0), Op2( 1), is32, - access_vm_reg, reset_unknown, c10_AMAIR1}, - - /* ICC_SGI1R */ - { CRm64(12), Op1( 0), is64, access_gic_sgi}, - - /* VBAR: swapped by interrupt.S. */ - { CRn(12), CRm( 0), Op1( 0), Op2( 0), is32, - NULL, reset_val, c12_VBAR, 0x00000000 }, - - /* ICC_ASGI1R */ - { CRm64(12), Op1( 1), is64, access_gic_sgi}, - /* ICC_SGI0R */ - { CRm64(12), Op1( 2), is64, access_gic_sgi}, - /* ICC_SRE */ - { CRn(12), CRm(12), Op1( 0), Op2(5), is32, access_gic_sre }, - - /* CONTEXTIDR/TPIDRURW/TPIDRURO/TPIDRPRW: swapped by interrupt.S. */ - { CRn(13), CRm( 0), Op1( 0), Op2( 1), is32, - access_vm_reg, reset_val, c13_CID, 0x00000000 }, - { CRn(13), CRm( 0), Op1( 0), Op2( 2), is32, - NULL, reset_unknown, c13_TID_URW }, - { CRn(13), CRm( 0), Op1( 0), Op2( 3), is32, - NULL, reset_unknown, c13_TID_URO }, - { CRn(13), CRm( 0), Op1( 0), Op2( 4), is32, - NULL, reset_unknown, c13_TID_PRIV }, - - /* CNTP */ - { CRm64(14), Op1( 2), is64, access_cntp_cval}, - - /* CNTKCTL: swapped by interrupt.S. */ - { CRn(14), CRm( 1), Op1( 0), Op2( 0), is32, - NULL, reset_val, c14_CNTKCTL, 0x00000000 }, - - /* CNTP */ - { CRn(14), CRm( 2), Op1( 0), Op2( 0), is32, access_cntp_tval }, - { CRn(14), CRm( 2), Op1( 0), Op2( 1), is32, access_cntp_ctl }, - - /* The Configuration Base Address Register. */ - { CRn(15), CRm( 0), Op1( 4), Op2( 0), is32, access_cbar}, -}; - -static int check_reg_table(const struct coproc_reg *table, unsigned int n) -{ - unsigned int i; - - for (i = 1; i < n; i++) { - if (cmp_reg(&table[i-1], &table[i]) >= 0) { - kvm_err("reg table %p out of order (%d)\n", table, i - 1); - return 1; - } - } - - return 0; -} - -/* Target specific emulation tables */ -static struct kvm_coproc_target_table *target_tables[KVM_ARM_NUM_TARGETS]; - -void kvm_register_target_coproc_table(struct kvm_coproc_target_table *table) -{ - BUG_ON(check_reg_table(table->table, table->num)); - target_tables[table->target] = table; -} - -/* Get specific register table for this target. */ -static const struct coproc_reg *get_target_table(unsigned target, size_t *num) -{ - struct kvm_coproc_target_table *table; - - table = target_tables[target]; - *num = table->num; - return table->table; -} - -#define reg_to_match_value(x) \ - ({ \ - unsigned long val; \ - val = (x)->CRn << 11; \ - val |= (x)->CRm << 7; \ - val |= (x)->Op1 << 4; \ - val |= (x)->Op2 << 1; \ - val |= !(x)->is_64bit; \ - val; \ - }) - -static int match_reg(const void *key, const void *elt) -{ - const unsigned long pval = (unsigned long)key; - const struct coproc_reg *r = elt; - - return pval - reg_to_match_value(r); -} - -static const struct coproc_reg *find_reg(const struct coproc_params *params, - const struct coproc_reg table[], - unsigned int num) -{ - unsigned long pval = reg_to_match_value(params); - - return bsearch((void *)pval, table, num, sizeof(table[0]), match_reg); -} - -static int emulate_cp15(struct kvm_vcpu *vcpu, - const struct coproc_params *params) -{ - size_t num; - const struct coproc_reg *table, *r; - - trace_kvm_emulate_cp15_imp(params->Op1, params->Rt1, params->CRn, - params->CRm, params->Op2, params->is_write); - - table = get_target_table(vcpu->arch.target, &num); - - /* Search target-specific then generic table. */ - r = find_reg(params, table, num); - if (!r) - r = find_reg(params, cp15_regs, ARRAY_SIZE(cp15_regs)); - - if (likely(r)) { - /* If we don't have an accessor, we should never get here! */ - BUG_ON(!r->access); - - if (likely(r->access(vcpu, params, r))) { - /* Skip instruction, since it was emulated */ - kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); - } - } else { - /* If access function fails, it should complain. */ - kvm_err("Unsupported guest CP15 access at: %08lx [%08lx]\n", - *vcpu_pc(vcpu), *vcpu_cpsr(vcpu)); - print_cp_instr(params); - kvm_inject_undefined(vcpu); - } - - return 1; -} - -static struct coproc_params decode_64bit_hsr(struct kvm_vcpu *vcpu) -{ - struct coproc_params params; - - params.CRn = (kvm_vcpu_get_hsr(vcpu) >> 1) & 0xf; - params.Rt1 = (kvm_vcpu_get_hsr(vcpu) >> 5) & 0xf; - params.is_write = ((kvm_vcpu_get_hsr(vcpu) & 1) == 0); - params.is_64bit = true; - - params.Op1 = (kvm_vcpu_get_hsr(vcpu) >> 16) & 0xf; - params.Op2 = 0; - params.Rt2 = (kvm_vcpu_get_hsr(vcpu) >> 10) & 0xf; - params.CRm = 0; - - return params; -} - -/** - * kvm_handle_cp15_64 -- handles a mrrc/mcrr trap on a guest CP15 access - * @vcpu: The VCPU pointer - * @run: The kvm_run struct - */ -int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - struct coproc_params params = decode_64bit_hsr(vcpu); - - return emulate_cp15(vcpu, ¶ms); -} - -/** - * kvm_handle_cp14_64 -- handles a mrrc/mcrr trap on a guest CP14 access - * @vcpu: The VCPU pointer - * @run: The kvm_run struct - */ -int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - struct coproc_params params = decode_64bit_hsr(vcpu); - - /* raz_wi cp14 */ - trap_raz_wi(vcpu, ¶ms, NULL); - - /* handled */ - kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); - return 1; -} - -static void reset_coproc_regs(struct kvm_vcpu *vcpu, - const struct coproc_reg *table, size_t num, - unsigned long *bmap) -{ - unsigned long i; - - for (i = 0; i < num; i++) - if (table[i].reset) { - int reg = table[i].reg; - - table[i].reset(vcpu, &table[i]); - if (reg > 0 && reg < NR_CP15_REGS) { - set_bit(reg, bmap); - if (table[i].is_64bit) - set_bit(reg + 1, bmap); - } - } -} - -static struct coproc_params decode_32bit_hsr(struct kvm_vcpu *vcpu) -{ - struct coproc_params params; - - params.CRm = (kvm_vcpu_get_hsr(vcpu) >> 1) & 0xf; - params.Rt1 = (kvm_vcpu_get_hsr(vcpu) >> 5) & 0xf; - params.is_write = ((kvm_vcpu_get_hsr(vcpu) & 1) == 0); - params.is_64bit = false; - - params.CRn = (kvm_vcpu_get_hsr(vcpu) >> 10) & 0xf; - params.Op1 = (kvm_vcpu_get_hsr(vcpu) >> 14) & 0x7; - params.Op2 = (kvm_vcpu_get_hsr(vcpu) >> 17) & 0x7; - params.Rt2 = 0; - - return params; -} - -/** - * kvm_handle_cp15_32 -- handles a mrc/mcr trap on a guest CP15 access - * @vcpu: The VCPU pointer - * @run: The kvm_run struct - */ -int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - struct coproc_params params = decode_32bit_hsr(vcpu); - return emulate_cp15(vcpu, ¶ms); -} - -/** - * kvm_handle_cp14_32 -- handles a mrc/mcr trap on a guest CP14 access - * @vcpu: The VCPU pointer - * @run: The kvm_run struct - */ -int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - struct coproc_params params = decode_32bit_hsr(vcpu); - - /* raz_wi cp14 */ - trap_raz_wi(vcpu, ¶ms, NULL); - - /* handled */ - kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); - return 1; -} - -/****************************************************************************** - * Userspace API - *****************************************************************************/ - -static bool index_to_params(u64 id, struct coproc_params *params) -{ - switch (id & KVM_REG_SIZE_MASK) { - case KVM_REG_SIZE_U32: - /* Any unused index bits means it's not valid. */ - if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK - | KVM_REG_ARM_COPROC_MASK - | KVM_REG_ARM_32_CRN_MASK - | KVM_REG_ARM_CRM_MASK - | KVM_REG_ARM_OPC1_MASK - | KVM_REG_ARM_32_OPC2_MASK)) - return false; - - params->is_64bit = false; - params->CRn = ((id & KVM_REG_ARM_32_CRN_MASK) - >> KVM_REG_ARM_32_CRN_SHIFT); - params->CRm = ((id & KVM_REG_ARM_CRM_MASK) - >> KVM_REG_ARM_CRM_SHIFT); - params->Op1 = ((id & KVM_REG_ARM_OPC1_MASK) - >> KVM_REG_ARM_OPC1_SHIFT); - params->Op2 = ((id & KVM_REG_ARM_32_OPC2_MASK) - >> KVM_REG_ARM_32_OPC2_SHIFT); - return true; - case KVM_REG_SIZE_U64: - /* Any unused index bits means it's not valid. */ - if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK - | KVM_REG_ARM_COPROC_MASK - | KVM_REG_ARM_CRM_MASK - | KVM_REG_ARM_OPC1_MASK)) - return false; - params->is_64bit = true; - /* CRm to CRn: see cp15_to_index for details */ - params->CRn = ((id & KVM_REG_ARM_CRM_MASK) - >> KVM_REG_ARM_CRM_SHIFT); - params->Op1 = ((id & KVM_REG_ARM_OPC1_MASK) - >> KVM_REG_ARM_OPC1_SHIFT); - params->Op2 = 0; - params->CRm = 0; - return true; - default: - return false; - } -} - -/* Decode an index value, and find the cp15 coproc_reg entry. */ -static const struct coproc_reg *index_to_coproc_reg(struct kvm_vcpu *vcpu, - u64 id) -{ - size_t num; - const struct coproc_reg *table, *r; - struct coproc_params params; - - /* We only do cp15 for now. */ - if ((id & KVM_REG_ARM_COPROC_MASK) >> KVM_REG_ARM_COPROC_SHIFT != 15) - return NULL; - - if (!index_to_params(id, ¶ms)) - return NULL; - - table = get_target_table(vcpu->arch.target, &num); - r = find_reg(¶ms, table, num); - if (!r) - r = find_reg(¶ms, cp15_regs, ARRAY_SIZE(cp15_regs)); - - /* Not saved in the cp15 array? */ - if (r && !r->reg) - r = NULL; - - return r; -} - -/* - * These are the invariant cp15 registers: we let the guest see the host - * versions of these, so they're part of the guest state. - * - * A future CPU may provide a mechanism to present different values to - * the guest, or a future kvm may trap them. - */ -/* Unfortunately, there's no register-argument for mrc, so generate. */ -#define FUNCTION_FOR32(crn, crm, op1, op2, name) \ - static void get_##name(struct kvm_vcpu *v, \ - const struct coproc_reg *r) \ - { \ - u32 val; \ - \ - asm volatile("mrc p15, " __stringify(op1) \ - ", %0, c" __stringify(crn) \ - ", c" __stringify(crm) \ - ", " __stringify(op2) "\n" : "=r" (val)); \ - ((struct coproc_reg *)r)->val = val; \ - } - -FUNCTION_FOR32(0, 0, 0, 0, MIDR) -FUNCTION_FOR32(0, 0, 0, 1, CTR) -FUNCTION_FOR32(0, 0, 0, 2, TCMTR) -FUNCTION_FOR32(0, 0, 0, 3, TLBTR) -FUNCTION_FOR32(0, 0, 0, 6, REVIDR) -FUNCTION_FOR32(0, 1, 0, 0, ID_PFR0) -FUNCTION_FOR32(0, 1, 0, 1, ID_PFR1) -FUNCTION_FOR32(0, 1, 0, 2, ID_DFR0) -FUNCTION_FOR32(0, 1, 0, 3, ID_AFR0) -FUNCTION_FOR32(0, 1, 0, 4, ID_MMFR0) -FUNCTION_FOR32(0, 1, 0, 5, ID_MMFR1) -FUNCTION_FOR32(0, 1, 0, 6, ID_MMFR2) -FUNCTION_FOR32(0, 1, 0, 7, ID_MMFR3) -FUNCTION_FOR32(0, 2, 0, 0, ID_ISAR0) -FUNCTION_FOR32(0, 2, 0, 1, ID_ISAR1) -FUNCTION_FOR32(0, 2, 0, 2, ID_ISAR2) -FUNCTION_FOR32(0, 2, 0, 3, ID_ISAR3) -FUNCTION_FOR32(0, 2, 0, 4, ID_ISAR4) -FUNCTION_FOR32(0, 2, 0, 5, ID_ISAR5) -FUNCTION_FOR32(0, 0, 1, 1, CLIDR) -FUNCTION_FOR32(0, 0, 1, 7, AIDR) - -/* ->val is filled in by kvm_invariant_coproc_table_init() */ -static struct coproc_reg invariant_cp15[] = { - { CRn( 0), CRm( 0), Op1( 0), Op2( 0), is32, NULL, get_MIDR }, - { CRn( 0), CRm( 0), Op1( 0), Op2( 1), is32, NULL, get_CTR }, - { CRn( 0), CRm( 0), Op1( 0), Op2( 2), is32, NULL, get_TCMTR }, - { CRn( 0), CRm( 0), Op1( 0), Op2( 3), is32, NULL, get_TLBTR }, - { CRn( 0), CRm( 0), Op1( 0), Op2( 6), is32, NULL, get_REVIDR }, - - { CRn( 0), CRm( 0), Op1( 1), Op2( 1), is32, NULL, get_CLIDR }, - { CRn( 0), CRm( 0), Op1( 1), Op2( 7), is32, NULL, get_AIDR }, - - { CRn( 0), CRm( 1), Op1( 0), Op2( 0), is32, NULL, get_ID_PFR0 }, - { CRn( 0), CRm( 1), Op1( 0), Op2( 1), is32, NULL, get_ID_PFR1 }, - { CRn( 0), CRm( 1), Op1( 0), Op2( 2), is32, NULL, get_ID_DFR0 }, - { CRn( 0), CRm( 1), Op1( 0), Op2( 3), is32, NULL, get_ID_AFR0 }, - { CRn( 0), CRm( 1), Op1( 0), Op2( 4), is32, NULL, get_ID_MMFR0 }, - { CRn( 0), CRm( 1), Op1( 0), Op2( 5), is32, NULL, get_ID_MMFR1 }, - { CRn( 0), CRm( 1), Op1( 0), Op2( 6), is32, NULL, get_ID_MMFR2 }, - { CRn( 0), CRm( 1), Op1( 0), Op2( 7), is32, NULL, get_ID_MMFR3 }, - - { CRn( 0), CRm( 2), Op1( 0), Op2( 0), is32, NULL, get_ID_ISAR0 }, - { CRn( 0), CRm( 2), Op1( 0), Op2( 1), is32, NULL, get_ID_ISAR1 }, - { CRn( 0), CRm( 2), Op1( 0), Op2( 2), is32, NULL, get_ID_ISAR2 }, - { CRn( 0), CRm( 2), Op1( 0), Op2( 3), is32, NULL, get_ID_ISAR3 }, - { CRn( 0), CRm( 2), Op1( 0), Op2( 4), is32, NULL, get_ID_ISAR4 }, - { CRn( 0), CRm( 2), Op1( 0), Op2( 5), is32, NULL, get_ID_ISAR5 }, -}; - -/* - * Reads a register value from a userspace address to a kernel - * variable. Make sure that register size matches sizeof(*__val). - */ -static int reg_from_user(void *val, const void __user *uaddr, u64 id) -{ - if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0) - return -EFAULT; - return 0; -} - -/* - * Writes a register value to a userspace address from a kernel variable. - * Make sure that register size matches sizeof(*__val). - */ -static int reg_to_user(void __user *uaddr, const void *val, u64 id) -{ - if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0) - return -EFAULT; - return 0; -} - -static int get_invariant_cp15(u64 id, void __user *uaddr) -{ - struct coproc_params params; - const struct coproc_reg *r; - int ret; - - if (!index_to_params(id, ¶ms)) - return -ENOENT; - - r = find_reg(¶ms, invariant_cp15, ARRAY_SIZE(invariant_cp15)); - if (!r) - return -ENOENT; - - ret = -ENOENT; - if (KVM_REG_SIZE(id) == 4) { - u32 val = r->val; - - ret = reg_to_user(uaddr, &val, id); - } else if (KVM_REG_SIZE(id) == 8) { - ret = reg_to_user(uaddr, &r->val, id); - } - return ret; -} - -static int set_invariant_cp15(u64 id, void __user *uaddr) -{ - struct coproc_params params; - const struct coproc_reg *r; - int err; - u64 val; - - if (!index_to_params(id, ¶ms)) - return -ENOENT; - r = find_reg(¶ms, invariant_cp15, ARRAY_SIZE(invariant_cp15)); - if (!r) - return -ENOENT; - - err = -ENOENT; - if (KVM_REG_SIZE(id) == 4) { - u32 val32; - - err = reg_from_user(&val32, uaddr, id); - if (!err) - val = val32; - } else if (KVM_REG_SIZE(id) == 8) { - err = reg_from_user(&val, uaddr, id); - } - if (err) - return err; - - /* This is what we mean by invariant: you can't change it. */ - if (r->val != val) - return -EINVAL; - - return 0; -} - -static bool is_valid_cache(u32 val) -{ - u32 level, ctype; - - if (val >= CSSELR_MAX) - return false; - - /* Bottom bit is Instruction or Data bit. Next 3 bits are level. */ - level = (val >> 1); - ctype = (cache_levels >> (level * 3)) & 7; - - switch (ctype) { - case 0: /* No cache */ - return false; - case 1: /* Instruction cache only */ - return (val & 1); - case 2: /* Data cache only */ - case 4: /* Unified cache */ - return !(val & 1); - case 3: /* Separate instruction and data caches */ - return true; - default: /* Reserved: we can't know instruction or data. */ - return false; - } -} - -/* Which cache CCSIDR represents depends on CSSELR value. */ -static u32 get_ccsidr(u32 csselr) -{ - u32 ccsidr; - - /* Make sure noone else changes CSSELR during this! */ - local_irq_disable(); - /* Put value into CSSELR */ - asm volatile("mcr p15, 2, %0, c0, c0, 0" : : "r" (csselr)); - isb(); - /* Read result out of CCSIDR */ - asm volatile("mrc p15, 1, %0, c0, c0, 0" : "=r" (ccsidr)); - local_irq_enable(); - - return ccsidr; -} - -static int demux_c15_get(u64 id, void __user *uaddr) -{ - u32 val; - u32 __user *uval = uaddr; - - /* Fail if we have unknown bits set. */ - if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK - | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1))) - return -ENOENT; - - switch (id & KVM_REG_ARM_DEMUX_ID_MASK) { - case KVM_REG_ARM_DEMUX_ID_CCSIDR: - if (KVM_REG_SIZE(id) != 4) - return -ENOENT; - val = (id & KVM_REG_ARM_DEMUX_VAL_MASK) - >> KVM_REG_ARM_DEMUX_VAL_SHIFT; - if (!is_valid_cache(val)) - return -ENOENT; - - return put_user(get_ccsidr(val), uval); - default: - return -ENOENT; - } -} - -static int demux_c15_set(u64 id, void __user *uaddr) -{ - u32 val, newval; - u32 __user *uval = uaddr; - - /* Fail if we have unknown bits set. */ - if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK - | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1))) - return -ENOENT; - - switch (id & KVM_REG_ARM_DEMUX_ID_MASK) { - case KVM_REG_ARM_DEMUX_ID_CCSIDR: - if (KVM_REG_SIZE(id) != 4) - return -ENOENT; - val = (id & KVM_REG_ARM_DEMUX_VAL_MASK) - >> KVM_REG_ARM_DEMUX_VAL_SHIFT; - if (!is_valid_cache(val)) - return -ENOENT; - - if (get_user(newval, uval)) - return -EFAULT; - - /* This is also invariant: you can't change it. */ - if (newval != get_ccsidr(val)) - return -EINVAL; - return 0; - default: - return -ENOENT; - } -} - -#ifdef CONFIG_VFPv3 -static const int vfp_sysregs[] = { KVM_REG_ARM_VFP_FPEXC, - KVM_REG_ARM_VFP_FPSCR, - KVM_REG_ARM_VFP_FPINST, - KVM_REG_ARM_VFP_FPINST2, - KVM_REG_ARM_VFP_MVFR0, - KVM_REG_ARM_VFP_MVFR1, - KVM_REG_ARM_VFP_FPSID }; - -static unsigned int num_fp_regs(void) -{ - if (((fmrx(MVFR0) & MVFR0_A_SIMD_MASK) >> MVFR0_A_SIMD_BIT) == 2) - return 32; - else - return 16; -} - -static unsigned int num_vfp_regs(void) -{ - /* Normal FP regs + control regs. */ - return num_fp_regs() + ARRAY_SIZE(vfp_sysregs); -} - -static int copy_vfp_regids(u64 __user *uindices) -{ - unsigned int i; - const u64 u32reg = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_VFP; - const u64 u64reg = KVM_REG_ARM | KVM_REG_SIZE_U64 | KVM_REG_ARM_VFP; - - for (i = 0; i < num_fp_regs(); i++) { - if (put_user((u64reg | KVM_REG_ARM_VFP_BASE_REG) + i, - uindices)) - return -EFAULT; - uindices++; - } - - for (i = 0; i < ARRAY_SIZE(vfp_sysregs); i++) { - if (put_user(u32reg | vfp_sysregs[i], uindices)) - return -EFAULT; - uindices++; - } - - return num_vfp_regs(); -} - -static int vfp_get_reg(const struct kvm_vcpu *vcpu, u64 id, void __user *uaddr) -{ - u32 vfpid = (id & KVM_REG_ARM_VFP_MASK); - u32 val; - - /* Fail if we have unknown bits set. */ - if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK - | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1))) - return -ENOENT; - - if (vfpid < num_fp_regs()) { - if (KVM_REG_SIZE(id) != 8) - return -ENOENT; - return reg_to_user(uaddr, &vcpu->arch.ctxt.vfp.fpregs[vfpid], - id); - } - - /* FP control registers are all 32 bit. */ - if (KVM_REG_SIZE(id) != 4) - return -ENOENT; - - switch (vfpid) { - case KVM_REG_ARM_VFP_FPEXC: - return reg_to_user(uaddr, &vcpu->arch.ctxt.vfp.fpexc, id); - case KVM_REG_ARM_VFP_FPSCR: - return reg_to_user(uaddr, &vcpu->arch.ctxt.vfp.fpscr, id); - case KVM_REG_ARM_VFP_FPINST: - return reg_to_user(uaddr, &vcpu->arch.ctxt.vfp.fpinst, id); - case KVM_REG_ARM_VFP_FPINST2: - return reg_to_user(uaddr, &vcpu->arch.ctxt.vfp.fpinst2, id); - case KVM_REG_ARM_VFP_MVFR0: - val = fmrx(MVFR0); - return reg_to_user(uaddr, &val, id); - case KVM_REG_ARM_VFP_MVFR1: - val = fmrx(MVFR1); - return reg_to_user(uaddr, &val, id); - case KVM_REG_ARM_VFP_FPSID: - val = fmrx(FPSID); - return reg_to_user(uaddr, &val, id); - default: - return -ENOENT; - } -} - -static int vfp_set_reg(struct kvm_vcpu *vcpu, u64 id, const void __user *uaddr) -{ - u32 vfpid = (id & KVM_REG_ARM_VFP_MASK); - u32 val; - - /* Fail if we have unknown bits set. */ - if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK - | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1))) - return -ENOENT; - - if (vfpid < num_fp_regs()) { - if (KVM_REG_SIZE(id) != 8) - return -ENOENT; - return reg_from_user(&vcpu->arch.ctxt.vfp.fpregs[vfpid], - uaddr, id); - } - - /* FP control registers are all 32 bit. */ - if (KVM_REG_SIZE(id) != 4) - return -ENOENT; - - switch (vfpid) { - case KVM_REG_ARM_VFP_FPEXC: - return reg_from_user(&vcpu->arch.ctxt.vfp.fpexc, uaddr, id); - case KVM_REG_ARM_VFP_FPSCR: - return reg_from_user(&vcpu->arch.ctxt.vfp.fpscr, uaddr, id); - case KVM_REG_ARM_VFP_FPINST: - return reg_from_user(&vcpu->arch.ctxt.vfp.fpinst, uaddr, id); - case KVM_REG_ARM_VFP_FPINST2: - return reg_from_user(&vcpu->arch.ctxt.vfp.fpinst2, uaddr, id); - /* These are invariant. */ - case KVM_REG_ARM_VFP_MVFR0: - if (reg_from_user(&val, uaddr, id)) - return -EFAULT; - if (val != fmrx(MVFR0)) - return -EINVAL; - return 0; - case KVM_REG_ARM_VFP_MVFR1: - if (reg_from_user(&val, uaddr, id)) - return -EFAULT; - if (val != fmrx(MVFR1)) - return -EINVAL; - return 0; - case KVM_REG_ARM_VFP_FPSID: - if (reg_from_user(&val, uaddr, id)) - return -EFAULT; - if (val != fmrx(FPSID)) - return -EINVAL; - return 0; - default: - return -ENOENT; - } -} -#else /* !CONFIG_VFPv3 */ -static unsigned int num_vfp_regs(void) -{ - return 0; -} - -static int copy_vfp_regids(u64 __user *uindices) -{ - return 0; -} - -static int vfp_get_reg(const struct kvm_vcpu *vcpu, u64 id, void __user *uaddr) -{ - return -ENOENT; -} - -static int vfp_set_reg(struct kvm_vcpu *vcpu, u64 id, const void __user *uaddr) -{ - return -ENOENT; -} -#endif /* !CONFIG_VFPv3 */ - -int kvm_arm_coproc_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - const struct coproc_reg *r; - void __user *uaddr = (void __user *)(long)reg->addr; - int ret; - - if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX) - return demux_c15_get(reg->id, uaddr); - - if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_VFP) - return vfp_get_reg(vcpu, reg->id, uaddr); - - r = index_to_coproc_reg(vcpu, reg->id); - if (!r) - return get_invariant_cp15(reg->id, uaddr); - - ret = -ENOENT; - if (KVM_REG_SIZE(reg->id) == 8) { - u64 val; - - val = vcpu_cp15_reg64_get(vcpu, r); - ret = reg_to_user(uaddr, &val, reg->id); - } else if (KVM_REG_SIZE(reg->id) == 4) { - ret = reg_to_user(uaddr, &vcpu_cp15(vcpu, r->reg), reg->id); - } - - return ret; -} - -int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - const struct coproc_reg *r; - void __user *uaddr = (void __user *)(long)reg->addr; - int ret; - - if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX) - return demux_c15_set(reg->id, uaddr); - - if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_VFP) - return vfp_set_reg(vcpu, reg->id, uaddr); - - r = index_to_coproc_reg(vcpu, reg->id); - if (!r) - return set_invariant_cp15(reg->id, uaddr); - - ret = -ENOENT; - if (KVM_REG_SIZE(reg->id) == 8) { - u64 val; - - ret = reg_from_user(&val, uaddr, reg->id); - if (!ret) - vcpu_cp15_reg64_set(vcpu, r, val); - } else if (KVM_REG_SIZE(reg->id) == 4) { - ret = reg_from_user(&vcpu_cp15(vcpu, r->reg), uaddr, reg->id); - } - - return ret; -} - -static unsigned int num_demux_regs(void) -{ - unsigned int i, count = 0; - - for (i = 0; i < CSSELR_MAX; i++) - if (is_valid_cache(i)) - count++; - - return count; -} - -static int write_demux_regids(u64 __user *uindices) -{ - u64 val = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX; - unsigned int i; - - val |= KVM_REG_ARM_DEMUX_ID_CCSIDR; - for (i = 0; i < CSSELR_MAX; i++) { - if (!is_valid_cache(i)) - continue; - if (put_user(val | i, uindices)) - return -EFAULT; - uindices++; - } - return 0; -} - -static u64 cp15_to_index(const struct coproc_reg *reg) -{ - u64 val = KVM_REG_ARM | (15 << KVM_REG_ARM_COPROC_SHIFT); - if (reg->is_64bit) { - val |= KVM_REG_SIZE_U64; - val |= (reg->Op1 << KVM_REG_ARM_OPC1_SHIFT); - /* - * CRn always denotes the primary coproc. reg. nr. for the - * in-kernel representation, but the user space API uses the - * CRm for the encoding, because it is modelled after the - * MRRC/MCRR instructions: see the ARM ARM rev. c page - * B3-1445 - */ - val |= (reg->CRn << KVM_REG_ARM_CRM_SHIFT); - } else { - val |= KVM_REG_SIZE_U32; - val |= (reg->Op1 << KVM_REG_ARM_OPC1_SHIFT); - val |= (reg->Op2 << KVM_REG_ARM_32_OPC2_SHIFT); - val |= (reg->CRm << KVM_REG_ARM_CRM_SHIFT); - val |= (reg->CRn << KVM_REG_ARM_32_CRN_SHIFT); - } - return val; -} - -static bool copy_reg_to_user(const struct coproc_reg *reg, u64 __user **uind) -{ - if (!*uind) - return true; - - if (put_user(cp15_to_index(reg), *uind)) - return false; - - (*uind)++; - return true; -} - -/* Assumed ordered tables, see kvm_coproc_table_init. */ -static int walk_cp15(struct kvm_vcpu *vcpu, u64 __user *uind) -{ - const struct coproc_reg *i1, *i2, *end1, *end2; - unsigned int total = 0; - size_t num; - - /* We check for duplicates here, to allow arch-specific overrides. */ - i1 = get_target_table(vcpu->arch.target, &num); - end1 = i1 + num; - i2 = cp15_regs; - end2 = cp15_regs + ARRAY_SIZE(cp15_regs); - - BUG_ON(i1 == end1 || i2 == end2); - - /* Walk carefully, as both tables may refer to the same register. */ - while (i1 || i2) { - int cmp = cmp_reg(i1, i2); - /* target-specific overrides generic entry. */ - if (cmp <= 0) { - /* Ignore registers we trap but don't save. */ - if (i1->reg) { - if (!copy_reg_to_user(i1, &uind)) - return -EFAULT; - total++; - } - } else { - /* Ignore registers we trap but don't save. */ - if (i2->reg) { - if (!copy_reg_to_user(i2, &uind)) - return -EFAULT; - total++; - } - } - - if (cmp <= 0 && ++i1 == end1) - i1 = NULL; - if (cmp >= 0 && ++i2 == end2) - i2 = NULL; - } - return total; -} - -unsigned long kvm_arm_num_coproc_regs(struct kvm_vcpu *vcpu) -{ - return ARRAY_SIZE(invariant_cp15) - + num_demux_regs() - + num_vfp_regs() - + walk_cp15(vcpu, (u64 __user *)NULL); -} - -int kvm_arm_copy_coproc_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) -{ - unsigned int i; - int err; - - /* Then give them all the invariant registers' indices. */ - for (i = 0; i < ARRAY_SIZE(invariant_cp15); i++) { - if (put_user(cp15_to_index(&invariant_cp15[i]), uindices)) - return -EFAULT; - uindices++; - } - - err = walk_cp15(vcpu, uindices); - if (err < 0) - return err; - uindices += err; - - err = copy_vfp_regids(uindices); - if (err < 0) - return err; - uindices += err; - - return write_demux_regids(uindices); -} - -void kvm_coproc_table_init(void) -{ - unsigned int i; - - /* Make sure tables are unique and in order. */ - BUG_ON(check_reg_table(cp15_regs, ARRAY_SIZE(cp15_regs))); - BUG_ON(check_reg_table(invariant_cp15, ARRAY_SIZE(invariant_cp15))); - - /* We abuse the reset function to overwrite the table itself. */ - for (i = 0; i < ARRAY_SIZE(invariant_cp15); i++) - invariant_cp15[i].reset(NULL, &invariant_cp15[i]); - - /* - * CLIDR format is awkward, so clean it up. See ARM B4.1.20: - * - * If software reads the Cache Type fields from Ctype1 - * upwards, once it has seen a value of 0b000, no caches - * exist at further-out levels of the hierarchy. So, for - * example, if Ctype3 is the first Cache Type field with a - * value of 0b000, the values of Ctype4 to Ctype7 must be - * ignored. - */ - asm volatile("mrc p15, 1, %0, c0, c0, 1" : "=r" (cache_levels)); - for (i = 0; i < 7; i++) - if (((cache_levels >> (i*3)) & 7) == 0) - break; - /* Clear all higher bits. */ - cache_levels &= (1 << (i*3))-1; -} - -/** - * kvm_reset_coprocs - sets cp15 registers to reset value - * @vcpu: The VCPU pointer - * - * This function finds the right table above and sets the registers on the - * virtual CPU struct to their architecturally defined reset values. - */ -void kvm_reset_coprocs(struct kvm_vcpu *vcpu) -{ - size_t num; - const struct coproc_reg *table; - DECLARE_BITMAP(bmap, NR_CP15_REGS) = { 0, }; - - /* Generic chip reset first (so target could override). */ - reset_coproc_regs(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs), bmap); - - table = get_target_table(vcpu->arch.target, &num); - reset_coproc_regs(vcpu, table, num, bmap); - - for (num = 1; num < NR_CP15_REGS; num++) - WARN(!test_bit(num, bmap), - "Didn't reset vcpu_cp15(vcpu, %zi)", num); -} diff --git a/arch/arm/kvm/coproc.h b/arch/arm/kvm/coproc.h deleted file mode 100644 index 637065b13012..000000000000 --- a/arch/arm/kvm/coproc.h +++ /dev/null @@ -1,130 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Authors: Christoffer Dall - */ - -#ifndef __ARM_KVM_COPROC_LOCAL_H__ -#define __ARM_KVM_COPROC_LOCAL_H__ - -struct coproc_params { - unsigned long CRn; - unsigned long CRm; - unsigned long Op1; - unsigned long Op2; - unsigned long Rt1; - unsigned long Rt2; - bool is_64bit; - bool is_write; -}; - -struct coproc_reg { - /* MRC/MCR/MRRC/MCRR instruction which accesses it. */ - unsigned long CRn; - unsigned long CRm; - unsigned long Op1; - unsigned long Op2; - - bool is_64bit; - - /* Trapped access from guest, if non-NULL. */ - bool (*access)(struct kvm_vcpu *, - const struct coproc_params *, - const struct coproc_reg *); - - /* Initialization for vcpu. */ - void (*reset)(struct kvm_vcpu *, const struct coproc_reg *); - - /* Index into vcpu_cp15(vcpu, ...), or 0 if we don't need to save it. */ - unsigned long reg; - - /* Value (usually reset value) */ - u64 val; -}; - -static inline void print_cp_instr(const struct coproc_params *p) -{ - /* Look, we even formatted it for you to paste into the table! */ - if (p->is_64bit) { - kvm_pr_unimpl(" { CRm64(%2lu), Op1(%2lu), is64, func_%s },\n", - p->CRn, p->Op1, p->is_write ? "write" : "read"); - } else { - kvm_pr_unimpl(" { CRn(%2lu), CRm(%2lu), Op1(%2lu), Op2(%2lu), is32," - " func_%s },\n", - p->CRn, p->CRm, p->Op1, p->Op2, - p->is_write ? "write" : "read"); - } -} - -static inline bool ignore_write(struct kvm_vcpu *vcpu, - const struct coproc_params *p) -{ - return true; -} - -static inline bool read_zero(struct kvm_vcpu *vcpu, - const struct coproc_params *p) -{ - *vcpu_reg(vcpu, p->Rt1) = 0; - return true; -} - -/* Reset functions */ -static inline void reset_unknown(struct kvm_vcpu *vcpu, - const struct coproc_reg *r) -{ - BUG_ON(!r->reg); - BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.ctxt.cp15)); - vcpu_cp15(vcpu, r->reg) = 0xdecafbad; -} - -static inline void reset_val(struct kvm_vcpu *vcpu, const struct coproc_reg *r) -{ - BUG_ON(!r->reg); - BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.ctxt.cp15)); - vcpu_cp15(vcpu, r->reg) = r->val; -} - -static inline void reset_unknown64(struct kvm_vcpu *vcpu, - const struct coproc_reg *r) -{ - BUG_ON(!r->reg); - BUG_ON(r->reg + 1 >= ARRAY_SIZE(vcpu->arch.ctxt.cp15)); - - vcpu_cp15(vcpu, r->reg) = 0xdecafbad; - vcpu_cp15(vcpu, r->reg+1) = 0xd0c0ffee; -} - -static inline int cmp_reg(const struct coproc_reg *i1, - const struct coproc_reg *i2) -{ - BUG_ON(i1 == i2); - if (!i1) - return 1; - else if (!i2) - return -1; - if (i1->CRn != i2->CRn) - return i1->CRn - i2->CRn; - if (i1->CRm != i2->CRm) - return i1->CRm - i2->CRm; - if (i1->Op1 != i2->Op1) - return i1->Op1 - i2->Op1; - if (i1->Op2 != i2->Op2) - return i1->Op2 - i2->Op2; - return i2->is_64bit - i1->is_64bit; -} - - -#define CRn(_x) .CRn = _x -#define CRm(_x) .CRm = _x -#define CRm64(_x) .CRn = _x, .CRm = 0 -#define Op1(_x) .Op1 = _x -#define Op2(_x) .Op2 = _x -#define is64 .is_64bit = true -#define is32 .is_64bit = false - -bool access_vm_reg(struct kvm_vcpu *vcpu, - const struct coproc_params *p, - const struct coproc_reg *r); - -#endif /* __ARM_KVM_COPROC_LOCAL_H__ */ diff --git a/arch/arm/kvm/coproc_a15.c b/arch/arm/kvm/coproc_a15.c deleted file mode 100644 index 36bf15421ae8..000000000000 --- a/arch/arm/kvm/coproc_a15.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Authors: Rusty Russell - * Christoffer Dall - */ -#include -#include -#include -#include - -#include "coproc.h" - -/* - * A15-specific CP15 registers. - * CRn denotes the primary register number, but is copied to the CRm in the - * user space API for 64-bit register access in line with the terminology used - * in the ARM ARM. - * Important: Must be sorted ascending by CRn, CRM, Op1, Op2 and with 64-bit - * registers preceding 32-bit ones. - */ -static const struct coproc_reg a15_regs[] = { - /* SCTLR: swapped by interrupt.S. */ - { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32, - access_vm_reg, reset_val, c1_SCTLR, 0x00C50078 }, -}; - -static struct kvm_coproc_target_table a15_target_table = { - .target = KVM_ARM_TARGET_CORTEX_A15, - .table = a15_regs, - .num = ARRAY_SIZE(a15_regs), -}; - -static int __init coproc_a15_init(void) -{ - kvm_register_target_coproc_table(&a15_target_table); - return 0; -} -late_initcall(coproc_a15_init); diff --git a/arch/arm/kvm/coproc_a7.c b/arch/arm/kvm/coproc_a7.c deleted file mode 100644 index 40f643e1e05c..000000000000 --- a/arch/arm/kvm/coproc_a7.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Copyright (C) 2013 - ARM Ltd - * - * Authors: Rusty Russell - * Christoffer Dall - * Jonathan Austin - */ -#include -#include -#include -#include - -#include "coproc.h" - -/* - * Cortex-A7 specific CP15 registers. - * CRn denotes the primary register number, but is copied to the CRm in the - * user space API for 64-bit register access in line with the terminology used - * in the ARM ARM. - * Important: Must be sorted ascending by CRn, CRM, Op1, Op2 and with 64-bit - * registers preceding 32-bit ones. - */ -static const struct coproc_reg a7_regs[] = { - /* SCTLR: swapped by interrupt.S. */ - { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32, - access_vm_reg, reset_val, c1_SCTLR, 0x00C50878 }, -}; - -static struct kvm_coproc_target_table a7_target_table = { - .target = KVM_ARM_TARGET_CORTEX_A7, - .table = a7_regs, - .num = ARRAY_SIZE(a7_regs), -}; - -static int __init coproc_a7_init(void) -{ - kvm_register_target_coproc_table(&a7_target_table); - return 0; -} -late_initcall(coproc_a7_init); diff --git a/arch/arm/kvm/emulate.c b/arch/arm/kvm/emulate.c deleted file mode 100644 index 29bb852140c5..000000000000 --- a/arch/arm/kvm/emulate.c +++ /dev/null @@ -1,166 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#include -#include -#include -#include -#include -#include - -#include "trace.h" - -#define VCPU_NR_MODES 6 -#define VCPU_REG_OFFSET_USR 0 -#define VCPU_REG_OFFSET_FIQ 1 -#define VCPU_REG_OFFSET_IRQ 2 -#define VCPU_REG_OFFSET_SVC 3 -#define VCPU_REG_OFFSET_ABT 4 -#define VCPU_REG_OFFSET_UND 5 -#define REG_OFFSET(_reg) \ - (offsetof(struct kvm_regs, _reg) / sizeof(u32)) - -#define USR_REG_OFFSET(_num) REG_OFFSET(usr_regs.uregs[_num]) - -static const unsigned long vcpu_reg_offsets[VCPU_NR_MODES][15] = { - /* USR/SYS Registers */ - [VCPU_REG_OFFSET_USR] = { - USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), - USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), - USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), - USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), - USR_REG_OFFSET(12), USR_REG_OFFSET(13), USR_REG_OFFSET(14), - }, - - /* FIQ Registers */ - [VCPU_REG_OFFSET_FIQ] = { - USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), - USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), - USR_REG_OFFSET(6), USR_REG_OFFSET(7), - REG_OFFSET(fiq_regs[0]), /* r8 */ - REG_OFFSET(fiq_regs[1]), /* r9 */ - REG_OFFSET(fiq_regs[2]), /* r10 */ - REG_OFFSET(fiq_regs[3]), /* r11 */ - REG_OFFSET(fiq_regs[4]), /* r12 */ - REG_OFFSET(fiq_regs[5]), /* r13 */ - REG_OFFSET(fiq_regs[6]), /* r14 */ - }, - - /* IRQ Registers */ - [VCPU_REG_OFFSET_IRQ] = { - USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), - USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), - USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), - USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), - USR_REG_OFFSET(12), - REG_OFFSET(irq_regs[0]), /* r13 */ - REG_OFFSET(irq_regs[1]), /* r14 */ - }, - - /* SVC Registers */ - [VCPU_REG_OFFSET_SVC] = { - USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), - USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), - USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), - USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), - USR_REG_OFFSET(12), - REG_OFFSET(svc_regs[0]), /* r13 */ - REG_OFFSET(svc_regs[1]), /* r14 */ - }, - - /* ABT Registers */ - [VCPU_REG_OFFSET_ABT] = { - USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), - USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), - USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), - USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), - USR_REG_OFFSET(12), - REG_OFFSET(abt_regs[0]), /* r13 */ - REG_OFFSET(abt_regs[1]), /* r14 */ - }, - - /* UND Registers */ - [VCPU_REG_OFFSET_UND] = { - USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), - USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), - USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), - USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), - USR_REG_OFFSET(12), - REG_OFFSET(und_regs[0]), /* r13 */ - REG_OFFSET(und_regs[1]), /* r14 */ - }, -}; - -/* - * Return a pointer to the register number valid in the current mode of - * the virtual CPU. - */ -unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num) -{ - unsigned long *reg_array = (unsigned long *)&vcpu->arch.ctxt.gp_regs; - unsigned long mode = *vcpu_cpsr(vcpu) & MODE_MASK; - - switch (mode) { - case USR_MODE...SVC_MODE: - mode &= ~MODE32_BIT; /* 0 ... 3 */ - break; - - case ABT_MODE: - mode = VCPU_REG_OFFSET_ABT; - break; - - case UND_MODE: - mode = VCPU_REG_OFFSET_UND; - break; - - case SYSTEM_MODE: - mode = VCPU_REG_OFFSET_USR; - break; - - default: - BUG(); - } - - return reg_array + vcpu_reg_offsets[mode][reg_num]; -} - -/* - * Return the SPSR for the current mode of the virtual CPU. - */ -unsigned long *__vcpu_spsr(struct kvm_vcpu *vcpu) -{ - unsigned long mode = *vcpu_cpsr(vcpu) & MODE_MASK; - switch (mode) { - case SVC_MODE: - return &vcpu->arch.ctxt.gp_regs.KVM_ARM_SVC_spsr; - case ABT_MODE: - return &vcpu->arch.ctxt.gp_regs.KVM_ARM_ABT_spsr; - case UND_MODE: - return &vcpu->arch.ctxt.gp_regs.KVM_ARM_UND_spsr; - case IRQ_MODE: - return &vcpu->arch.ctxt.gp_regs.KVM_ARM_IRQ_spsr; - case FIQ_MODE: - return &vcpu->arch.ctxt.gp_regs.KVM_ARM_FIQ_spsr; - default: - BUG(); - } -} - -/****************************************************************************** - * Inject exceptions into the guest - */ - -/** - * kvm_inject_vabt - inject an async abort / SError into the guest - * @vcpu: The VCPU to receive the exception - * - * It is assumed that this code is called from the VCPU thread and that the - * VCPU therefore is not currently executing guest code. - */ -void kvm_inject_vabt(struct kvm_vcpu *vcpu) -{ - *vcpu_hcr(vcpu) |= HCR_VA; -} diff --git a/arch/arm/kvm/guest.c b/arch/arm/kvm/guest.c deleted file mode 100644 index 9f7ae0d8690f..000000000000 --- a/arch/arm/kvm/guest.c +++ /dev/null @@ -1,387 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM } -#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU } - -struct kvm_stats_debugfs_item debugfs_entries[] = { - VCPU_STAT(halt_successful_poll), - VCPU_STAT(halt_attempted_poll), - VCPU_STAT(halt_poll_invalid), - VCPU_STAT(halt_wakeup), - VCPU_STAT(hvc_exit_stat), - VCPU_STAT(wfe_exit_stat), - VCPU_STAT(wfi_exit_stat), - VCPU_STAT(mmio_exit_user), - VCPU_STAT(mmio_exit_kernel), - VCPU_STAT(exits), - { NULL } -}; - -static u64 core_reg_offset_from_id(u64 id) -{ - return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE); -} - -static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - u32 __user *uaddr = (u32 __user *)(long)reg->addr; - struct kvm_regs *regs = &vcpu->arch.ctxt.gp_regs; - u64 off; - - if (KVM_REG_SIZE(reg->id) != 4) - return -ENOENT; - - /* Our ID is an index into the kvm_regs struct. */ - off = core_reg_offset_from_id(reg->id); - if (off >= sizeof(*regs) / KVM_REG_SIZE(reg->id)) - return -ENOENT; - - return put_user(((u32 *)regs)[off], uaddr); -} - -static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - u32 __user *uaddr = (u32 __user *)(long)reg->addr; - struct kvm_regs *regs = &vcpu->arch.ctxt.gp_regs; - u64 off, val; - - if (KVM_REG_SIZE(reg->id) != 4) - return -ENOENT; - - /* Our ID is an index into the kvm_regs struct. */ - off = core_reg_offset_from_id(reg->id); - if (off >= sizeof(*regs) / KVM_REG_SIZE(reg->id)) - return -ENOENT; - - if (get_user(val, uaddr) != 0) - return -EFAULT; - - if (off == KVM_REG_ARM_CORE_REG(usr_regs.ARM_cpsr)) { - unsigned long mode = val & MODE_MASK; - switch (mode) { - case USR_MODE: - case FIQ_MODE: - case IRQ_MODE: - case SVC_MODE: - case ABT_MODE: - case UND_MODE: - break; - default: - return -EINVAL; - } - } - - ((u32 *)regs)[off] = val; - return 0; -} - -int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - return -EINVAL; -} - -int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - return -EINVAL; -} - -#define NUM_TIMER_REGS 3 - -static bool is_timer_reg(u64 index) -{ - switch (index) { - case KVM_REG_ARM_TIMER_CTL: - case KVM_REG_ARM_TIMER_CNT: - case KVM_REG_ARM_TIMER_CVAL: - return true; - } - return false; -} - -static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) -{ - if (put_user(KVM_REG_ARM_TIMER_CTL, uindices)) - return -EFAULT; - uindices++; - if (put_user(KVM_REG_ARM_TIMER_CNT, uindices)) - return -EFAULT; - uindices++; - if (put_user(KVM_REG_ARM_TIMER_CVAL, uindices)) - return -EFAULT; - - return 0; -} - -static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - void __user *uaddr = (void __user *)(long)reg->addr; - u64 val; - int ret; - - ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id)); - if (ret != 0) - return -EFAULT; - - return kvm_arm_timer_set_reg(vcpu, reg->id, val); -} - -static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - void __user *uaddr = (void __user *)(long)reg->addr; - u64 val; - - val = kvm_arm_timer_get_reg(vcpu, reg->id); - return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0; -} - -static unsigned long num_core_regs(void) -{ - return sizeof(struct kvm_regs) / sizeof(u32); -} - -/** - * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG - * - * This is for all registers. - */ -unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) -{ - return num_core_regs() + kvm_arm_num_coproc_regs(vcpu) - + kvm_arm_get_fw_num_regs(vcpu) - + NUM_TIMER_REGS; -} - -/** - * kvm_arm_copy_reg_indices - get indices of all registers. - * - * We do core registers right here, then we append coproc regs. - */ -int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) -{ - unsigned int i; - const u64 core_reg = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_CORE; - int ret; - - for (i = 0; i < sizeof(struct kvm_regs)/sizeof(u32); i++) { - if (put_user(core_reg | i, uindices)) - return -EFAULT; - uindices++; - } - - ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices); - if (ret) - return ret; - uindices += kvm_arm_get_fw_num_regs(vcpu); - - ret = copy_timer_indices(vcpu, uindices); - if (ret) - return ret; - uindices += NUM_TIMER_REGS; - - return kvm_arm_copy_coproc_indices(vcpu, uindices); -} - -int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - /* We currently use nothing arch-specific in upper 32 bits */ - if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM >> 32) - return -EINVAL; - - /* Register group 16 means we want a core register. */ - if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_CORE) - return get_core_reg(vcpu, reg); - - if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_FW) - return kvm_arm_get_fw_reg(vcpu, reg); - - if (is_timer_reg(reg->id)) - return get_timer_reg(vcpu, reg); - - return kvm_arm_coproc_get_reg(vcpu, reg); -} - -int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - /* We currently use nothing arch-specific in upper 32 bits */ - if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM >> 32) - return -EINVAL; - - /* Register group 16 means we set a core register. */ - if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_CORE) - return set_core_reg(vcpu, reg); - - if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_FW) - return kvm_arm_set_fw_reg(vcpu, reg); - - if (is_timer_reg(reg->id)) - return set_timer_reg(vcpu, reg); - - return kvm_arm_coproc_set_reg(vcpu, reg); -} - -int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - return -EINVAL; -} - -int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - return -EINVAL; -} - - -int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, - struct kvm_vcpu_events *events) -{ - events->exception.serror_pending = !!(*vcpu_hcr(vcpu) & HCR_VA); - - /* - * We never return a pending ext_dabt here because we deliver it to - * the virtual CPU directly when setting the event and it's no longer - * 'pending' at this point. - */ - - return 0; -} - -int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, - struct kvm_vcpu_events *events) -{ - bool serror_pending = events->exception.serror_pending; - bool has_esr = events->exception.serror_has_esr; - bool ext_dabt_pending = events->exception.ext_dabt_pending; - - if (serror_pending && has_esr) - return -EINVAL; - else if (serror_pending) - kvm_inject_vabt(vcpu); - - if (ext_dabt_pending) - kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); - - return 0; -} - -int __attribute_const__ kvm_target_cpu(void) -{ - switch (read_cpuid_part()) { - case ARM_CPU_PART_CORTEX_A7: - return KVM_ARM_TARGET_CORTEX_A7; - case ARM_CPU_PART_CORTEX_A15: - return KVM_ARM_TARGET_CORTEX_A15; - default: - return -EINVAL; - } -} - -int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) -{ - int target = kvm_target_cpu(); - - if (target < 0) - return -ENODEV; - - memset(init, 0, sizeof(*init)); - - /* - * For now, we don't return any features. - * In future, we might use features to return target - * specific features available for the preferred - * target type. - */ - init->target = (__u32)target; - - return 0; -} - -int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) -{ - return -EINVAL; -} - -int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) -{ - return -EINVAL; -} - -int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, - struct kvm_translation *tr) -{ - return -EINVAL; -} - -int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, - struct kvm_guest_debug *dbg) -{ - return -EINVAL; -} - -int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, - struct kvm_device_attr *attr) -{ - int ret; - - switch (attr->group) { - case KVM_ARM_VCPU_TIMER_CTRL: - ret = kvm_arm_timer_set_attr(vcpu, attr); - break; - default: - ret = -ENXIO; - break; - } - - return ret; -} - -int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, - struct kvm_device_attr *attr) -{ - int ret; - - switch (attr->group) { - case KVM_ARM_VCPU_TIMER_CTRL: - ret = kvm_arm_timer_get_attr(vcpu, attr); - break; - default: - ret = -ENXIO; - break; - } - - return ret; -} - -int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, - struct kvm_device_attr *attr) -{ - int ret; - - switch (attr->group) { - case KVM_ARM_VCPU_TIMER_CTRL: - ret = kvm_arm_timer_has_attr(vcpu, attr); - break; - default: - ret = -ENXIO; - break; - } - - return ret; -} diff --git a/arch/arm/kvm/handle_exit.c b/arch/arm/kvm/handle_exit.c deleted file mode 100644 index e58a89d2f13f..000000000000 --- a/arch/arm/kvm/handle_exit.c +++ /dev/null @@ -1,175 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "trace.h" - -typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *); - -static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - int ret; - - trace_kvm_hvc(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0), - kvm_vcpu_hvc_get_imm(vcpu)); - vcpu->stat.hvc_exit_stat++; - - ret = kvm_hvc_call_handler(vcpu); - if (ret < 0) { - vcpu_set_reg(vcpu, 0, ~0UL); - return 1; - } - - return ret; -} - -static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - /* - * "If an SMC instruction executed at Non-secure EL1 is - * trapped to EL2 because HCR_EL2.TSC is 1, the exception is a - * Trap exception, not a Secure Monitor Call exception [...]" - * - * We need to advance the PC after the trap, as it would - * otherwise return to the same address... - */ - vcpu_set_reg(vcpu, 0, ~0UL); - kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); - return 1; -} - -/** - * kvm_handle_wfx - handle a WFI or WFE instructions trapped in guests - * @vcpu: the vcpu pointer - * @run: the kvm_run structure pointer - * - * WFE: Yield the CPU and come back to this vcpu when the scheduler - * decides to. - * WFI: Simply call kvm_vcpu_block(), which will halt execution of - * world-switches and schedule other host processes until there is an - * incoming IRQ or FIQ to the VM. - */ -static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - if (kvm_vcpu_get_hsr(vcpu) & HSR_WFI_IS_WFE) { - trace_kvm_wfx(*vcpu_pc(vcpu), true); - vcpu->stat.wfe_exit_stat++; - kvm_vcpu_on_spin(vcpu, vcpu_mode_priv(vcpu)); - } else { - trace_kvm_wfx(*vcpu_pc(vcpu), false); - vcpu->stat.wfi_exit_stat++; - kvm_vcpu_block(vcpu); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); - } - - kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); - - return 1; -} - -static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - u32 hsr = kvm_vcpu_get_hsr(vcpu); - - kvm_pr_unimpl("Unknown exception class: hsr: %#08x\n", - hsr); - - kvm_inject_undefined(vcpu); - return 1; -} - -static exit_handle_fn arm_exit_handlers[] = { - [0 ... HSR_EC_MAX] = kvm_handle_unknown_ec, - [HSR_EC_WFI] = kvm_handle_wfx, - [HSR_EC_CP15_32] = kvm_handle_cp15_32, - [HSR_EC_CP15_64] = kvm_handle_cp15_64, - [HSR_EC_CP14_MR] = kvm_handle_cp14_32, - [HSR_EC_CP14_LS] = kvm_handle_cp14_load_store, - [HSR_EC_CP14_64] = kvm_handle_cp14_64, - [HSR_EC_CP_0_13] = kvm_handle_cp_0_13_access, - [HSR_EC_CP10_ID] = kvm_handle_cp10_id, - [HSR_EC_HVC] = handle_hvc, - [HSR_EC_SMC] = handle_smc, - [HSR_EC_IABT] = kvm_handle_guest_abort, - [HSR_EC_DABT] = kvm_handle_guest_abort, -}; - -static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) -{ - u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu); - - return arm_exit_handlers[hsr_ec]; -} - -/* - * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on - * proper exit to userspace. - */ -int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, - int exception_index) -{ - exit_handle_fn exit_handler; - - if (ARM_ABORT_PENDING(exception_index)) { - u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu); - - /* - * HVC/SMC already have an adjusted PC, which we need - * to correct in order to return to after having - * injected the abort. - */ - if (hsr_ec == HSR_EC_HVC || hsr_ec == HSR_EC_SMC) { - u32 adj = kvm_vcpu_trap_il_is32bit(vcpu) ? 4 : 2; - *vcpu_pc(vcpu) -= adj; - } - - kvm_inject_vabt(vcpu); - return 1; - } - - exception_index = ARM_EXCEPTION_CODE(exception_index); - - switch (exception_index) { - case ARM_EXCEPTION_IRQ: - return 1; - case ARM_EXCEPTION_HVC: - /* - * See ARM ARM B1.14.1: "Hyp traps on instructions - * that fail their condition code check" - */ - if (!kvm_condition_valid(vcpu)) { - kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); - return 1; - } - - exit_handler = kvm_get_exit_handler(vcpu); - - return exit_handler(vcpu, run); - case ARM_EXCEPTION_DATA_ABORT: - kvm_inject_vabt(vcpu); - return 1; - case ARM_EXCEPTION_HYP_GONE: - /* - * HYP has been reset to the hyp-stub. This happens - * when a guest is pre-empted by kvm_reboot()'s - * shutdown call. - */ - run->exit_reason = KVM_EXIT_FAIL_ENTRY; - return 0; - default: - kvm_pr_unimpl("Unsupported exception type: %d", - exception_index); - run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - return 0; - } -} diff --git a/arch/arm/kvm/hyp/Makefile b/arch/arm/kvm/hyp/Makefile deleted file mode 100644 index ba88b1eca93c..000000000000 --- a/arch/arm/kvm/hyp/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for Kernel-based Virtual Machine module, HYP part -# - -ccflags-y += -fno-stack-protector -DDISABLE_BRANCH_PROFILING - -KVM=../../../../virt/kvm - -CFLAGS_ARMV7VE :=$(call cc-option, -march=armv7ve) - -obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/vgic-v3-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/timer-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/aarch32.o - -obj-$(CONFIG_KVM_ARM_HOST) += tlb.o -obj-$(CONFIG_KVM_ARM_HOST) += cp15-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += vfp.o -obj-$(CONFIG_KVM_ARM_HOST) += banked-sr.o -CFLAGS_banked-sr.o += $(CFLAGS_ARMV7VE) - -obj-$(CONFIG_KVM_ARM_HOST) += entry.o -obj-$(CONFIG_KVM_ARM_HOST) += hyp-entry.o -obj-$(CONFIG_KVM_ARM_HOST) += switch.o -CFLAGS_switch.o += $(CFLAGS_ARMV7VE) -obj-$(CONFIG_KVM_ARM_HOST) += s2-setup.o - -# KVM code is run at a different exception code with a different map, so -# compiler instrumentation that inserts callbacks or checks into the code may -# cause crashes. Just disable it. -GCOV_PROFILE := n -KASAN_SANITIZE := n -UBSAN_SANITIZE := n -KCOV_INSTRUMENT := n diff --git a/arch/arm/kvm/hyp/banked-sr.c b/arch/arm/kvm/hyp/banked-sr.c deleted file mode 100644 index c4632ed9e819..000000000000 --- a/arch/arm/kvm/hyp/banked-sr.c +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Original code: - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - * - * Mostly rewritten in C by Marc Zyngier - */ - -#include - -/* - * gcc before 4.9 doesn't understand -march=armv7ve, so we have to - * trick the assembler. - */ -__asm__(".arch_extension virt"); - -void __hyp_text __banked_save_state(struct kvm_cpu_context *ctxt) -{ - ctxt->gp_regs.usr_regs.ARM_sp = read_special(SP_usr); - ctxt->gp_regs.usr_regs.ARM_pc = read_special(ELR_hyp); - ctxt->gp_regs.usr_regs.ARM_cpsr = read_special(SPSR); - ctxt->gp_regs.KVM_ARM_SVC_sp = read_special(SP_svc); - ctxt->gp_regs.KVM_ARM_SVC_lr = read_special(LR_svc); - ctxt->gp_regs.KVM_ARM_SVC_spsr = read_special(SPSR_svc); - ctxt->gp_regs.KVM_ARM_ABT_sp = read_special(SP_abt); - ctxt->gp_regs.KVM_ARM_ABT_lr = read_special(LR_abt); - ctxt->gp_regs.KVM_ARM_ABT_spsr = read_special(SPSR_abt); - ctxt->gp_regs.KVM_ARM_UND_sp = read_special(SP_und); - ctxt->gp_regs.KVM_ARM_UND_lr = read_special(LR_und); - ctxt->gp_regs.KVM_ARM_UND_spsr = read_special(SPSR_und); - ctxt->gp_regs.KVM_ARM_IRQ_sp = read_special(SP_irq); - ctxt->gp_regs.KVM_ARM_IRQ_lr = read_special(LR_irq); - ctxt->gp_regs.KVM_ARM_IRQ_spsr = read_special(SPSR_irq); - ctxt->gp_regs.KVM_ARM_FIQ_r8 = read_special(R8_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_r9 = read_special(R9_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_r10 = read_special(R10_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_fp = read_special(R11_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_ip = read_special(R12_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_sp = read_special(SP_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_lr = read_special(LR_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_spsr = read_special(SPSR_fiq); -} - -void __hyp_text __banked_restore_state(struct kvm_cpu_context *ctxt) -{ - write_special(ctxt->gp_regs.usr_regs.ARM_sp, SP_usr); - write_special(ctxt->gp_regs.usr_regs.ARM_pc, ELR_hyp); - write_special(ctxt->gp_regs.usr_regs.ARM_cpsr, SPSR_cxsf); - write_special(ctxt->gp_regs.KVM_ARM_SVC_sp, SP_svc); - write_special(ctxt->gp_regs.KVM_ARM_SVC_lr, LR_svc); - write_special(ctxt->gp_regs.KVM_ARM_SVC_spsr, SPSR_svc); - write_special(ctxt->gp_regs.KVM_ARM_ABT_sp, SP_abt); - write_special(ctxt->gp_regs.KVM_ARM_ABT_lr, LR_abt); - write_special(ctxt->gp_regs.KVM_ARM_ABT_spsr, SPSR_abt); - write_special(ctxt->gp_regs.KVM_ARM_UND_sp, SP_und); - write_special(ctxt->gp_regs.KVM_ARM_UND_lr, LR_und); - write_special(ctxt->gp_regs.KVM_ARM_UND_spsr, SPSR_und); - write_special(ctxt->gp_regs.KVM_ARM_IRQ_sp, SP_irq); - write_special(ctxt->gp_regs.KVM_ARM_IRQ_lr, LR_irq); - write_special(ctxt->gp_regs.KVM_ARM_IRQ_spsr, SPSR_irq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_r8, R8_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_r9, R9_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_r10, R10_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_fp, R11_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_ip, R12_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_sp, SP_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_lr, LR_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_spsr, SPSR_fiq); -} diff --git a/arch/arm/kvm/hyp/cp15-sr.c b/arch/arm/kvm/hyp/cp15-sr.c deleted file mode 100644 index e6923306f698..000000000000 --- a/arch/arm/kvm/hyp/cp15-sr.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Original code: - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - * - * Mostly rewritten in C by Marc Zyngier - */ - -#include - -static u64 *cp15_64(struct kvm_cpu_context *ctxt, int idx) -{ - return (u64 *)(ctxt->cp15 + idx); -} - -void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt) -{ - ctxt->cp15[c0_CSSELR] = read_sysreg(CSSELR); - ctxt->cp15[c1_SCTLR] = read_sysreg(SCTLR); - ctxt->cp15[c1_CPACR] = read_sysreg(CPACR); - *cp15_64(ctxt, c2_TTBR0) = read_sysreg(TTBR0); - *cp15_64(ctxt, c2_TTBR1) = read_sysreg(TTBR1); - ctxt->cp15[c2_TTBCR] = read_sysreg(TTBCR); - ctxt->cp15[c3_DACR] = read_sysreg(DACR); - ctxt->cp15[c5_DFSR] = read_sysreg(DFSR); - ctxt->cp15[c5_IFSR] = read_sysreg(IFSR); - ctxt->cp15[c5_ADFSR] = read_sysreg(ADFSR); - ctxt->cp15[c5_AIFSR] = read_sysreg(AIFSR); - ctxt->cp15[c6_DFAR] = read_sysreg(DFAR); - ctxt->cp15[c6_IFAR] = read_sysreg(IFAR); - *cp15_64(ctxt, c7_PAR) = read_sysreg(PAR); - ctxt->cp15[c10_PRRR] = read_sysreg(PRRR); - ctxt->cp15[c10_NMRR] = read_sysreg(NMRR); - ctxt->cp15[c10_AMAIR0] = read_sysreg(AMAIR0); - ctxt->cp15[c10_AMAIR1] = read_sysreg(AMAIR1); - ctxt->cp15[c12_VBAR] = read_sysreg(VBAR); - ctxt->cp15[c13_CID] = read_sysreg(CID); - ctxt->cp15[c13_TID_URW] = read_sysreg(TID_URW); - ctxt->cp15[c13_TID_URO] = read_sysreg(TID_URO); - ctxt->cp15[c13_TID_PRIV] = read_sysreg(TID_PRIV); - ctxt->cp15[c14_CNTKCTL] = read_sysreg(CNTKCTL); -} - -void __hyp_text __sysreg_restore_state(struct kvm_cpu_context *ctxt) -{ - write_sysreg(ctxt->cp15[c0_MPIDR], VMPIDR); - write_sysreg(ctxt->cp15[c0_CSSELR], CSSELR); - write_sysreg(ctxt->cp15[c1_SCTLR], SCTLR); - write_sysreg(ctxt->cp15[c1_CPACR], CPACR); - write_sysreg(*cp15_64(ctxt, c2_TTBR0), TTBR0); - write_sysreg(*cp15_64(ctxt, c2_TTBR1), TTBR1); - write_sysreg(ctxt->cp15[c2_TTBCR], TTBCR); - write_sysreg(ctxt->cp15[c3_DACR], DACR); - write_sysreg(ctxt->cp15[c5_DFSR], DFSR); - write_sysreg(ctxt->cp15[c5_IFSR], IFSR); - write_sysreg(ctxt->cp15[c5_ADFSR], ADFSR); - write_sysreg(ctxt->cp15[c5_AIFSR], AIFSR); - write_sysreg(ctxt->cp15[c6_DFAR], DFAR); - write_sysreg(ctxt->cp15[c6_IFAR], IFAR); - write_sysreg(*cp15_64(ctxt, c7_PAR), PAR); - write_sysreg(ctxt->cp15[c10_PRRR], PRRR); - write_sysreg(ctxt->cp15[c10_NMRR], NMRR); - write_sysreg(ctxt->cp15[c10_AMAIR0], AMAIR0); - write_sysreg(ctxt->cp15[c10_AMAIR1], AMAIR1); - write_sysreg(ctxt->cp15[c12_VBAR], VBAR); - write_sysreg(ctxt->cp15[c13_CID], CID); - write_sysreg(ctxt->cp15[c13_TID_URW], TID_URW); - write_sysreg(ctxt->cp15[c13_TID_URO], TID_URO); - write_sysreg(ctxt->cp15[c13_TID_PRIV], TID_PRIV); - write_sysreg(ctxt->cp15[c14_CNTKCTL], CNTKCTL); -} diff --git a/arch/arm/kvm/hyp/entry.S b/arch/arm/kvm/hyp/entry.S deleted file mode 100644 index 4bd1f6a74180..000000000000 --- a/arch/arm/kvm/hyp/entry.S +++ /dev/null @@ -1,121 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2016 - ARM Ltd - * Author: Marc Zyngier -*/ - -#include -#include -#include -#include - - .arch_extension virt - - .text - .pushsection .hyp.text, "ax" - -#define USR_REGS_OFFSET (CPU_CTXT_GP_REGS + GP_REGS_USR) - -/* int __guest_enter(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host) */ -ENTRY(__guest_enter) - @ Save host registers - add r1, r1, #(USR_REGS_OFFSET + S_R4) - stm r1!, {r4-r12} - str lr, [r1, #4] @ Skip SP_usr (already saved) - - @ Restore guest registers - add r0, r0, #(VCPU_GUEST_CTXT + USR_REGS_OFFSET + S_R0) - ldr lr, [r0, #S_LR] - ldm r0, {r0-r12} - - clrex - eret -ENDPROC(__guest_enter) - -ENTRY(__guest_exit) - /* - * return convention: - * guest r0, r1, r2 saved on the stack - * r0: vcpu pointer - * r1: exception code - */ - - add r2, r0, #(VCPU_GUEST_CTXT + USR_REGS_OFFSET + S_R3) - stm r2!, {r3-r12} - str lr, [r2, #4] - add r2, r0, #(VCPU_GUEST_CTXT + USR_REGS_OFFSET + S_R0) - pop {r3, r4, r5} @ r0, r1, r2 - stm r2, {r3-r5} - - ldr r0, [r0, #VCPU_HOST_CTXT] - add r0, r0, #(USR_REGS_OFFSET + S_R4) - ldm r0!, {r4-r12} - ldr lr, [r0, #4] - - mov r0, r1 - mrs r1, SPSR - mrs r2, ELR_hyp - mrc p15, 4, r3, c5, c2, 0 @ HSR - - /* - * Force loads and stores to complete before unmasking aborts - * and forcing the delivery of the exception. This gives us a - * single instruction window, which the handler will try to - * match. - */ - dsb sy - cpsie a - - .global abort_guest_exit_start -abort_guest_exit_start: - - isb - - .global abort_guest_exit_end -abort_guest_exit_end: - - /* - * If we took an abort, r0[31] will be set, and cmp will set - * the N bit in PSTATE. - */ - cmp r0, #0 - msrmi SPSR_cxsf, r1 - msrmi ELR_hyp, r2 - mcrmi p15, 4, r3, c5, c2, 0 @ HSR - - bx lr -ENDPROC(__guest_exit) - -/* - * If VFPv3 support is not available, then we will not switch the VFP - * registers; however cp10 and cp11 accesses will still trap and fallback - * to the regular coprocessor emulation code, which currently will - * inject an undefined exception to the guest. - */ -#ifdef CONFIG_VFPv3 -ENTRY(__vfp_guest_restore) - push {r3, r4, lr} - - @ NEON/VFP used. Turn on VFP access. - mrc p15, 4, r1, c1, c1, 2 @ HCPTR - bic r1, r1, #(HCPTR_TCP(10) | HCPTR_TCP(11)) - mcr p15, 4, r1, c1, c1, 2 @ HCPTR - isb - - @ Switch VFP/NEON hardware state to the guest's - mov r4, r0 - ldr r0, [r0, #VCPU_HOST_CTXT] - add r0, r0, #CPU_CTXT_VFP - bl __vfp_save_state - add r0, r4, #(VCPU_GUEST_CTXT + CPU_CTXT_VFP) - bl __vfp_restore_state - - pop {r3, r4, lr} - pop {r0, r1, r2} - clrex - eret -ENDPROC(__vfp_guest_restore) -#endif - - .popsection - diff --git a/arch/arm/kvm/hyp/hyp-entry.S b/arch/arm/kvm/hyp/hyp-entry.S deleted file mode 100644 index fe3d7811a908..000000000000 --- a/arch/arm/kvm/hyp/hyp-entry.S +++ /dev/null @@ -1,295 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#include -#include -#include -#include - - .arch_extension virt - - .text - .pushsection .hyp.text, "ax" - -.macro load_vcpu reg - mrc p15, 4, \reg, c13, c0, 2 @ HTPIDR -.endm - -/******************************************************************** - * Hypervisor exception vector and handlers - * - * - * The KVM/ARM Hypervisor ABI is defined as follows: - * - * Entry to Hyp mode from the host kernel will happen _only_ when an HVC - * instruction is issued since all traps are disabled when running the host - * kernel as per the Hyp-mode initialization at boot time. - * - * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc - * below) when the HVC instruction is called from SVC mode (i.e. a guest or the - * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC - * instructions are called from within Hyp-mode. - * - * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode): - * Switching to Hyp mode is done through a simple HVC #0 instruction. The - * exception vector code will check that the HVC comes from VMID==0. - * - r0 contains a pointer to a HYP function - * - r1, r2, and r3 contain arguments to the above function. - * - The HYP function will be called with its arguments in r0, r1 and r2. - * On HYP function return, we return directly to SVC. - * - * Note that the above is used to execute code in Hyp-mode from a host-kernel - * point of view, and is a different concept from performing a world-switch and - * executing guest code SVC mode (with a VMID != 0). - */ - - .align 5 -__kvm_hyp_vector: - .global __kvm_hyp_vector - - @ Hyp-mode exception vector - W(b) hyp_reset - W(b) hyp_undef - W(b) hyp_svc - W(b) hyp_pabt - W(b) hyp_dabt - W(b) hyp_hvc - W(b) hyp_irq - W(b) hyp_fiq - -#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR - .align 5 -__kvm_hyp_vector_ic_inv: - .global __kvm_hyp_vector_ic_inv - - /* - * We encode the exception entry in the bottom 3 bits of - * SP, and we have to guarantee to be 8 bytes aligned. - */ - W(add) sp, sp, #1 /* Reset 7 */ - W(add) sp, sp, #1 /* Undef 6 */ - W(add) sp, sp, #1 /* Syscall 5 */ - W(add) sp, sp, #1 /* Prefetch abort 4 */ - W(add) sp, sp, #1 /* Data abort 3 */ - W(add) sp, sp, #1 /* HVC 2 */ - W(add) sp, sp, #1 /* IRQ 1 */ - W(nop) /* FIQ 0 */ - - mcr p15, 0, r0, c7, c5, 0 /* ICIALLU */ - isb - - b decode_vectors - - .align 5 -__kvm_hyp_vector_bp_inv: - .global __kvm_hyp_vector_bp_inv - - /* - * We encode the exception entry in the bottom 3 bits of - * SP, and we have to guarantee to be 8 bytes aligned. - */ - W(add) sp, sp, #1 /* Reset 7 */ - W(add) sp, sp, #1 /* Undef 6 */ - W(add) sp, sp, #1 /* Syscall 5 */ - W(add) sp, sp, #1 /* Prefetch abort 4 */ - W(add) sp, sp, #1 /* Data abort 3 */ - W(add) sp, sp, #1 /* HVC 2 */ - W(add) sp, sp, #1 /* IRQ 1 */ - W(nop) /* FIQ 0 */ - - mcr p15, 0, r0, c7, c5, 6 /* BPIALL */ - isb - -decode_vectors: - -#ifdef CONFIG_THUMB2_KERNEL - /* - * Yet another silly hack: Use VPIDR as a temp register. - * Thumb2 is really a pain, as SP cannot be used with most - * of the bitwise instructions. The vect_br macro ensures - * things gets cleaned-up. - */ - mcr p15, 4, r0, c0, c0, 0 /* VPIDR */ - mov r0, sp - and r0, r0, #7 - sub sp, sp, r0 - push {r1, r2} - mov r1, r0 - mrc p15, 4, r0, c0, c0, 0 /* VPIDR */ - mrc p15, 0, r2, c0, c0, 0 /* MIDR */ - mcr p15, 4, r2, c0, c0, 0 /* VPIDR */ -#endif - -.macro vect_br val, targ -ARM( eor sp, sp, #\val ) -ARM( tst sp, #7 ) -ARM( eorne sp, sp, #\val ) - -THUMB( cmp r1, #\val ) -THUMB( popeq {r1, r2} ) - - beq \targ -.endm - - vect_br 0, hyp_fiq - vect_br 1, hyp_irq - vect_br 2, hyp_hvc - vect_br 3, hyp_dabt - vect_br 4, hyp_pabt - vect_br 5, hyp_svc - vect_br 6, hyp_undef - vect_br 7, hyp_reset -#endif - -.macro invalid_vector label, cause - .align -\label: mov r0, #\cause - b __hyp_panic -.endm - - invalid_vector hyp_reset ARM_EXCEPTION_RESET - invalid_vector hyp_undef ARM_EXCEPTION_UNDEFINED - invalid_vector hyp_svc ARM_EXCEPTION_SOFTWARE - invalid_vector hyp_pabt ARM_EXCEPTION_PREF_ABORT - invalid_vector hyp_fiq ARM_EXCEPTION_FIQ - -ENTRY(__hyp_do_panic) - mrs lr, cpsr - bic lr, lr, #MODE_MASK - orr lr, lr, #SVC_MODE -THUMB( orr lr, lr, #PSR_T_BIT ) - msr spsr_cxsf, lr - ldr lr, =panic - msr ELR_hyp, lr - ldr lr, =__kvm_call_hyp - clrex - eret -ENDPROC(__hyp_do_panic) - -hyp_hvc: - /* - * Getting here is either because of a trap from a guest, - * or from executing HVC from the host kernel, which means - * "do something in Hyp mode". - */ - push {r0, r1, r2} - - @ Check syndrome register - mrc p15, 4, r1, c5, c2, 0 @ HSR - lsr r0, r1, #HSR_EC_SHIFT - cmp r0, #HSR_EC_HVC - bne guest_trap @ Not HVC instr. - - /* - * Let's check if the HVC came from VMID 0 and allow simple - * switch to Hyp mode - */ - mrrc p15, 6, r0, r2, c2 - lsr r2, r2, #16 - and r2, r2, #0xff - cmp r2, #0 - bne guest_hvc_trap @ Guest called HVC - - /* - * Getting here means host called HVC, we shift parameters and branch - * to Hyp function. - */ - pop {r0, r1, r2} - - /* - * Check if we have a kernel function, which is guaranteed to be - * bigger than the maximum hyp stub hypercall - */ - cmp r0, #HVC_STUB_HCALL_NR - bhs 1f - - /* - * Not a kernel function, treat it as a stub hypercall. - * Compute the physical address for __kvm_handle_stub_hvc - * (as the code lives in the idmaped page) and branch there. - * We hijack ip (r12) as a tmp register. - */ - push {r1} - ldr r1, =kimage_voffset - ldr r1, [r1] - ldr ip, =__kvm_handle_stub_hvc - sub ip, ip, r1 - pop {r1} - - bx ip - -1: - /* - * Pushing r2 here is just a way of keeping the stack aligned to - * 8 bytes on any path that can trigger a HYP exception. Here, - * we may well be about to jump into the guest, and the guest - * exit would otherwise be badly decoded by our fancy - * "decode-exception-without-a-branch" code... - */ - push {r2, lr} - - mov lr, r0 - mov r0, r1 - mov r1, r2 - mov r2, r3 - -THUMB( orr lr, #1) - blx lr @ Call the HYP function - - pop {r2, lr} - eret - -guest_hvc_trap: - movw r2, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1 - movt r2, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1 - ldr r0, [sp] @ Guest's r0 - teq r0, r2 - bne guest_trap - add sp, sp, #12 - @ Returns: - @ r0 = 0 - @ r1 = HSR value (perfectly predictable) - @ r2 = ARM_SMCCC_ARCH_WORKAROUND_1 - mov r0, #0 - eret - -guest_trap: - load_vcpu r0 @ Load VCPU pointer to r0 - -#ifdef CONFIG_VFPv3 - @ Check for a VFP access - lsr r1, r1, #HSR_EC_SHIFT - cmp r1, #HSR_EC_CP_0_13 - beq __vfp_guest_restore -#endif - - mov r1, #ARM_EXCEPTION_HVC - b __guest_exit - -hyp_irq: - push {r0, r1, r2} - mov r1, #ARM_EXCEPTION_IRQ - load_vcpu r0 @ Load VCPU pointer to r0 - b __guest_exit - -hyp_dabt: - push {r0, r1} - mrs r0, ELR_hyp - ldr r1, =abort_guest_exit_start -THUMB( add r1, r1, #1) - cmp r0, r1 - ldrne r1, =abort_guest_exit_end -THUMB( addne r1, r1, #1) - cmpne r0, r1 - pop {r0, r1} - bne __hyp_panic - - orr r0, r0, #(1 << ARM_EXIT_WITH_ABORT_BIT) - eret - - .ltorg - - .popsection diff --git a/arch/arm/kvm/hyp/s2-setup.c b/arch/arm/kvm/hyp/s2-setup.c deleted file mode 100644 index 5dfbea5adf65..000000000000 --- a/arch/arm/kvm/hyp/s2-setup.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2016 - ARM Ltd - * Author: Marc Zyngier - */ - -#include -#include -#include -#include - -void __hyp_text __init_stage2_translation(void) -{ - u64 val; - - val = read_sysreg(VTCR) & ~VTCR_MASK; - - val |= read_sysreg(HTCR) & VTCR_HTCR_SH; - val |= KVM_VTCR_SL0 | KVM_VTCR_T0SZ | KVM_VTCR_S; - - write_sysreg(val, VTCR); -} diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c deleted file mode 100644 index 1efeef3fd0ee..000000000000 --- a/arch/arm/kvm/hyp/switch.c +++ /dev/null @@ -1,242 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2015 - ARM Ltd - * Author: Marc Zyngier - */ -#include - -#include -#include -#include - -__asm__(".arch_extension virt"); - -/* - * Activate the traps, saving the host's fpexc register before - * overwriting it. We'll restore it on VM exit. - */ -static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu, u32 *fpexc_host) -{ - u32 val; - - /* - * We are about to set HCPTR.TCP10/11 to trap all floating point - * register accesses to HYP, however, the ARM ARM clearly states that - * traps are only taken to HYP if the operation would not otherwise - * trap to SVC. Therefore, always make sure that for 32-bit guests, - * we set FPEXC.EN to prevent traps to SVC, when setting the TCP bits. - */ - val = read_sysreg(VFP_FPEXC); - *fpexc_host = val; - if (!(val & FPEXC_EN)) { - write_sysreg(val | FPEXC_EN, VFP_FPEXC); - isb(); - } - - write_sysreg(vcpu->arch.hcr, HCR); - /* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */ - write_sysreg(HSTR_T(15), HSTR); - write_sysreg(HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11), HCPTR); - val = read_sysreg(HDCR); - val |= HDCR_TPM | HDCR_TPMCR; /* trap performance monitors */ - val |= HDCR_TDRA | HDCR_TDOSA | HDCR_TDA; /* trap debug regs */ - write_sysreg(val, HDCR); -} - -static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu) -{ - u32 val; - - /* - * If we pended a virtual abort, preserve it until it gets - * cleared. See B1.9.9 (Virtual Abort exception) for details, - * but the crucial bit is the zeroing of HCR.VA in the - * pseudocode. - */ - if (vcpu->arch.hcr & HCR_VA) - vcpu->arch.hcr = read_sysreg(HCR); - - write_sysreg(0, HCR); - write_sysreg(0, HSTR); - val = read_sysreg(HDCR); - write_sysreg(val & ~(HDCR_TPM | HDCR_TPMCR), HDCR); - write_sysreg(0, HCPTR); -} - -static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu) -{ - struct kvm *kvm = kern_hyp_va(vcpu->kvm); - write_sysreg(kvm_get_vttbr(kvm), VTTBR); - write_sysreg(vcpu->arch.midr, VPIDR); -} - -static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu) -{ - write_sysreg(0, VTTBR); - write_sysreg(read_sysreg(MIDR), VPIDR); -} - - -static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu) -{ - if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { - __vgic_v3_save_state(vcpu); - __vgic_v3_deactivate_traps(vcpu); - } -} - -static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu) -{ - if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { - __vgic_v3_activate_traps(vcpu); - __vgic_v3_restore_state(vcpu); - } -} - -static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu) -{ - u32 hsr = read_sysreg(HSR); - u8 ec = hsr >> HSR_EC_SHIFT; - u32 hpfar, far; - - vcpu->arch.fault.hsr = hsr; - - if (ec == HSR_EC_IABT) - far = read_sysreg(HIFAR); - else if (ec == HSR_EC_DABT) - far = read_sysreg(HDFAR); - else - return true; - - /* - * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode: - * - * Abort on the stage 2 translation for a memory access from a - * Non-secure PL1 or PL0 mode: - * - * For any Access flag fault or Translation fault, and also for any - * Permission fault on the stage 2 translation of a memory access - * made as part of a translation table walk for a stage 1 translation, - * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR - * is UNKNOWN. - */ - if (!(hsr & HSR_DABT_S1PTW) && (hsr & HSR_FSC_TYPE) == FSC_PERM) { - u64 par, tmp; - - par = read_sysreg(PAR); - write_sysreg(far, ATS1CPR); - isb(); - - tmp = read_sysreg(PAR); - write_sysreg(par, PAR); - - if (unlikely(tmp & 1)) - return false; /* Translation failed, back to guest */ - - hpfar = ((tmp >> 12) & ((1UL << 28) - 1)) << 4; - } else { - hpfar = read_sysreg(HPFAR); - } - - vcpu->arch.fault.hxfar = far; - vcpu->arch.fault.hpfar = hpfar; - return true; -} - -int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu) -{ - struct kvm_cpu_context *host_ctxt; - struct kvm_cpu_context *guest_ctxt; - bool fp_enabled; - u64 exit_code; - u32 fpexc; - - vcpu = kern_hyp_va(vcpu); - write_sysreg(vcpu, HTPIDR); - - host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); - guest_ctxt = &vcpu->arch.ctxt; - - __sysreg_save_state(host_ctxt); - __banked_save_state(host_ctxt); - - __activate_traps(vcpu, &fpexc); - __activate_vm(vcpu); - - __vgic_restore_state(vcpu); - __timer_enable_traps(vcpu); - - __sysreg_restore_state(guest_ctxt); - __banked_restore_state(guest_ctxt); - - /* Jump in the fire! */ -again: - exit_code = __guest_enter(vcpu, host_ctxt); - /* And we're baaack! */ - - if (exit_code == ARM_EXCEPTION_HVC && !__populate_fault_info(vcpu)) - goto again; - - fp_enabled = __vfp_enabled(); - - __banked_save_state(guest_ctxt); - __sysreg_save_state(guest_ctxt); - __timer_disable_traps(vcpu); - - __vgic_save_state(vcpu); - - __deactivate_traps(vcpu); - __deactivate_vm(vcpu); - - __banked_restore_state(host_ctxt); - __sysreg_restore_state(host_ctxt); - - if (fp_enabled) { - __vfp_save_state(&guest_ctxt->vfp); - __vfp_restore_state(&host_ctxt->vfp); - } - - write_sysreg(fpexc, VFP_FPEXC); - - return exit_code; -} - -static const char * const __hyp_panic_string[] = { - [ARM_EXCEPTION_RESET] = "\nHYP panic: RST PC:%08x CPSR:%08x", - [ARM_EXCEPTION_UNDEFINED] = "\nHYP panic: UNDEF PC:%08x CPSR:%08x", - [ARM_EXCEPTION_SOFTWARE] = "\nHYP panic: SVC PC:%08x CPSR:%08x", - [ARM_EXCEPTION_PREF_ABORT] = "\nHYP panic: PABRT PC:%08x CPSR:%08x", - [ARM_EXCEPTION_DATA_ABORT] = "\nHYP panic: DABRT PC:%08x ADDR:%08x", - [ARM_EXCEPTION_IRQ] = "\nHYP panic: IRQ PC:%08x CPSR:%08x", - [ARM_EXCEPTION_FIQ] = "\nHYP panic: FIQ PC:%08x CPSR:%08x", - [ARM_EXCEPTION_HVC] = "\nHYP panic: HVC PC:%08x CPSR:%08x", -}; - -void __hyp_text __noreturn __hyp_panic(int cause) -{ - u32 elr = read_special(ELR_hyp); - u32 val; - - if (cause == ARM_EXCEPTION_DATA_ABORT) - val = read_sysreg(HDFAR); - else - val = read_special(SPSR); - - if (read_sysreg(VTTBR)) { - struct kvm_vcpu *vcpu; - struct kvm_cpu_context *host_ctxt; - - vcpu = (struct kvm_vcpu *)read_sysreg(HTPIDR); - host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); - __timer_disable_traps(vcpu); - __deactivate_traps(vcpu); - __deactivate_vm(vcpu); - __banked_restore_state(host_ctxt); - __sysreg_restore_state(host_ctxt); - } - - /* Call panic for real */ - __hyp_do_panic(__hyp_panic_string[cause], elr, val); - - unreachable(); -} diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c deleted file mode 100644 index 848f27bbad9d..000000000000 --- a/arch/arm/kvm/hyp/tlb.c +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Original code: - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - * - * Mostly rewritten in C by Marc Zyngier - */ - -#include -#include - -/** - * Flush per-VMID TLBs - * - * __kvm_tlb_flush_vmid(struct kvm *kvm); - * - * We rely on the hardware to broadcast the TLB invalidation to all CPUs - * inside the inner-shareable domain (which is the case for all v7 - * implementations). If we come across a non-IS SMP implementation, we'll - * have to use an IPI based mechanism. Until then, we stick to the simple - * hardware assisted version. - * - * As v7 does not support flushing per IPA, just nuke the whole TLB - * instead, ignoring the ipa value. - */ -void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm) -{ - dsb(ishst); - - /* Switch to requested VMID */ - kvm = kern_hyp_va(kvm); - write_sysreg(kvm_get_vttbr(kvm), VTTBR); - isb(); - - write_sysreg(0, TLBIALLIS); - dsb(ish); - isb(); - - write_sysreg(0, VTTBR); -} - -void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa) -{ - __kvm_tlb_flush_vmid(kvm); -} - -void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu) -{ - struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm); - - /* Switch to requested VMID */ - write_sysreg(kvm_get_vttbr(kvm), VTTBR); - isb(); - - write_sysreg(0, TLBIALL); - dsb(nsh); - isb(); - - write_sysreg(0, VTTBR); -} - -void __hyp_text __kvm_flush_vm_context(void) -{ - write_sysreg(0, TLBIALLNSNHIS); - write_sysreg(0, ICIALLUIS); - dsb(ish); -} diff --git a/arch/arm/kvm/hyp/vfp.S b/arch/arm/kvm/hyp/vfp.S deleted file mode 100644 index 675a52348d8d..000000000000 --- a/arch/arm/kvm/hyp/vfp.S +++ /dev/null @@ -1,57 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#include -#include - - .text - .pushsection .hyp.text, "ax" - -/* void __vfp_save_state(struct vfp_hard_struct *vfp); */ -ENTRY(__vfp_save_state) - push {r4, r5} - VFPFMRX r1, FPEXC - - @ Make sure *really* VFP is enabled so we can touch the registers. - orr r5, r1, #FPEXC_EN - tst r5, #FPEXC_EX @ Check for VFP Subarchitecture - bic r5, r5, #FPEXC_EX @ FPEXC_EX disable - VFPFMXR FPEXC, r5 - isb - - VFPFMRX r2, FPSCR - beq 1f - - @ If FPEXC_EX is 0, then FPINST/FPINST2 reads are upredictable, so - @ we only need to save them if FPEXC_EX is set. - VFPFMRX r3, FPINST - tst r5, #FPEXC_FP2V - VFPFMRX r4, FPINST2, ne @ vmrsne -1: - VFPFSTMIA r0, r5 @ Save VFP registers - stm r0, {r1-r4} @ Save FPEXC, FPSCR, FPINST, FPINST2 - pop {r4, r5} - bx lr -ENDPROC(__vfp_save_state) - -/* void __vfp_restore_state(struct vfp_hard_struct *vfp); - * Assume FPEXC_EN is on and FPEXC_EX is off */ -ENTRY(__vfp_restore_state) - VFPFLDMIA r0, r1 @ Load VFP registers - ldm r0, {r0-r3} @ Load FPEXC, FPSCR, FPINST, FPINST2 - - VFPFMXR FPSCR, r1 - tst r0, #FPEXC_EX @ Check for VFP Subarchitecture - beq 1f - VFPFMXR FPINST, r2 - tst r0, #FPEXC_FP2V - VFPFMXR FPINST2, r3, ne -1: - VFPFMXR FPEXC, r0 @ FPEXC (last, in case !EN) - bx lr -ENDPROC(__vfp_restore_state) - - .popsection diff --git a/arch/arm/kvm/init.S b/arch/arm/kvm/init.S deleted file mode 100644 index 33e34b6d24b2..000000000000 --- a/arch/arm/kvm/init.S +++ /dev/null @@ -1,157 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/******************************************************************** - * Hypervisor initialization - * - should be called with: - * r0 = top of Hyp stack (kernel VA) - * r1 = pointer to hyp vectors - * r2,r3 = Hypervisor pgd pointer - * - * The init scenario is: - * - We jump in HYP with 3 parameters: runtime HYP pgd, runtime stack, - * runtime vectors - * - Invalidate TLBs - * - Set stack and vectors - * - Setup the page tables - * - Enable the MMU - * - Profit! (or eret, if you only care about the code). - * - * Another possibility is to get a HYP stub hypercall. - * We discriminate between the two by checking if r0 contains a value - * that is less than HVC_STUB_HCALL_NR. - */ - - .text - .pushsection .hyp.idmap.text,"ax" - .align 5 -__kvm_hyp_init: - .globl __kvm_hyp_init - - @ Hyp-mode exception vector - W(b) . - W(b) . - W(b) . - W(b) . - W(b) . - W(b) __do_hyp_init - W(b) . - W(b) . - -__do_hyp_init: - @ Check for a stub hypercall - cmp r0, #HVC_STUB_HCALL_NR - blo __kvm_handle_stub_hvc - - @ Set stack pointer - mov sp, r0 - - @ Set HVBAR to point to the HYP vectors - mcr p15, 4, r1, c12, c0, 0 @ HVBAR - - @ Set the HTTBR to point to the hypervisor PGD pointer passed - mcrr p15, 4, rr_lo_hi(r2, r3), c2 - - @ Set the HTCR and VTCR to the same shareability and cacheability - @ settings as the non-secure TTBCR and with T0SZ == 0. - mrc p15, 4, r0, c2, c0, 2 @ HTCR - ldr r2, =HTCR_MASK - bic r0, r0, r2 - mrc p15, 0, r1, c2, c0, 2 @ TTBCR - and r1, r1, #(HTCR_MASK & ~TTBCR_T0SZ) - orr r0, r0, r1 - mcr p15, 4, r0, c2, c0, 2 @ HTCR - - @ Use the same memory attributes for hyp. accesses as the kernel - @ (copy MAIRx ro HMAIRx). - mrc p15, 0, r0, c10, c2, 0 - mcr p15, 4, r0, c10, c2, 0 - mrc p15, 0, r0, c10, c2, 1 - mcr p15, 4, r0, c10, c2, 1 - - @ Invalidate the stale TLBs from Bootloader - mcr p15, 4, r0, c8, c7, 0 @ TLBIALLH - dsb ish - - @ Set the HSCTLR to: - @ - ARM/THUMB exceptions: Kernel config (Thumb-2 kernel) - @ - Endianness: Kernel config - @ - Fast Interrupt Features: Kernel config - @ - Write permission implies XN: disabled - @ - Instruction cache: enabled - @ - Data/Unified cache: enabled - @ - MMU: enabled (this code must be run from an identity mapping) - mrc p15, 4, r0, c1, c0, 0 @ HSCR - ldr r2, =HSCTLR_MASK - bic r0, r0, r2 - mrc p15, 0, r1, c1, c0, 0 @ SCTLR - ldr r2, =(HSCTLR_EE | HSCTLR_FI | HSCTLR_I | HSCTLR_C) - and r1, r1, r2 - ARM( ldr r2, =(HSCTLR_M) ) - THUMB( ldr r2, =(HSCTLR_M | HSCTLR_TE) ) - orr r1, r1, r2 - orr r0, r0, r1 - mcr p15, 4, r0, c1, c0, 0 @ HSCR - isb - - eret - -ENTRY(__kvm_handle_stub_hvc) - cmp r0, #HVC_SOFT_RESTART - bne 1f - - /* The target is expected in r1 */ - msr ELR_hyp, r1 - mrs r0, cpsr - bic r0, r0, #MODE_MASK - orr r0, r0, #HYP_MODE -THUMB( orr r0, r0, #PSR_T_BIT ) - msr spsr_cxsf, r0 - b reset - -1: cmp r0, #HVC_RESET_VECTORS - bne 1f - -reset: - /* We're now in idmap, disable MMU */ - mrc p15, 4, r1, c1, c0, 0 @ HSCTLR - ldr r0, =(HSCTLR_M | HSCTLR_A | HSCTLR_C | HSCTLR_I) - bic r1, r1, r0 - mcr p15, 4, r1, c1, c0, 0 @ HSCTLR - - /* - * Install stub vectors, using ardb's VA->PA trick. - */ -0: adr r0, 0b @ PA(0) - movw r1, #:lower16:__hyp_stub_vectors - 0b @ VA(stub) - VA(0) - movt r1, #:upper16:__hyp_stub_vectors - 0b - add r1, r1, r0 @ PA(stub) - mcr p15, 4, r1, c12, c0, 0 @ HVBAR - b exit - -1: ldr r0, =HVC_STUB_ERR - eret - -exit: - mov r0, #0 - eret -ENDPROC(__kvm_handle_stub_hvc) - - .ltorg - - .globl __kvm_hyp_init_end -__kvm_hyp_init_end: - - .popsection diff --git a/arch/arm/kvm/interrupts.S b/arch/arm/kvm/interrupts.S deleted file mode 100644 index 064f4f118ca7..000000000000 --- a/arch/arm/kvm/interrupts.S +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ - -#include - - .text - -/******************************************************************** - * Call function in Hyp mode - * - * - * unsigned long kvm_call_hyp(void *hypfn, ...); - * - * This is not really a variadic function in the classic C-way and care must - * be taken when calling this to ensure parameters are passed in registers - * only, since the stack will change between the caller and the callee. - * - * Call the function with the first argument containing a pointer to the - * function you wish to call in Hyp mode, and subsequent arguments will be - * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the - * function pointer can be passed). The function being called must be mapped - * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are - * passed in r0 (strictly 32bit). - * - * The calling convention follows the standard AAPCS: - * r0 - r3: caller save - * r12: caller save - * rest: callee save - */ -ENTRY(__kvm_call_hyp) - hvc #0 - bx lr -ENDPROC(__kvm_call_hyp) diff --git a/arch/arm/kvm/irq.h b/arch/arm/kvm/irq.h deleted file mode 100644 index 0d257de42c10..000000000000 --- a/arch/arm/kvm/irq.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * irq.h: in kernel interrupt controller related definitions - * Copyright (c) 2016 Red Hat, Inc. - * - * This header is included by irqchip.c. However, on ARM, interrupt - * controller declarations are located in include/kvm/arm_vgic.h since - * they are mostly shared between arm and arm64. - */ - -#ifndef __IRQ_H -#define __IRQ_H - -#include - -#endif diff --git a/arch/arm/kvm/reset.c b/arch/arm/kvm/reset.c deleted file mode 100644 index eb4174f6ebbd..000000000000 --- a/arch/arm/kvm/reset.c +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -/****************************************************************************** - * Cortex-A15 and Cortex-A7 Reset Values - */ - -static struct kvm_regs cortexa_regs_reset = { - .usr_regs.ARM_cpsr = SVC_MODE | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT, -}; - - -/******************************************************************************* - * Exported reset function - */ - -/** - * kvm_reset_vcpu - sets core registers and cp15 registers to reset value - * @vcpu: The VCPU pointer - * - * This function finds the right table above and sets the registers on the - * virtual CPU struct to their architecturally defined reset values. - */ -int kvm_reset_vcpu(struct kvm_vcpu *vcpu) -{ - struct kvm_regs *reset_regs; - - switch (vcpu->arch.target) { - case KVM_ARM_TARGET_CORTEX_A7: - case KVM_ARM_TARGET_CORTEX_A15: - reset_regs = &cortexa_regs_reset; - vcpu->arch.midr = read_cpuid_id(); - break; - default: - return -ENODEV; - } - - /* Reset core registers */ - memcpy(&vcpu->arch.ctxt.gp_regs, reset_regs, sizeof(vcpu->arch.ctxt.gp_regs)); - - /* Reset CP15 registers */ - kvm_reset_coprocs(vcpu); - - /* - * Additional reset state handling that PSCI may have imposed on us. - * Must be done after all the sys_reg reset. - */ - if (READ_ONCE(vcpu->arch.reset_state.reset)) { - unsigned long target_pc = vcpu->arch.reset_state.pc; - - /* Gracefully handle Thumb2 entry point */ - if (target_pc & 1) { - target_pc &= ~1UL; - vcpu_set_thumb(vcpu); - } - - /* Propagate caller endianness */ - if (vcpu->arch.reset_state.be) - kvm_vcpu_set_be(vcpu); - - *vcpu_pc(vcpu) = target_pc; - vcpu_set_reg(vcpu, 0, vcpu->arch.reset_state.r0); - - vcpu->arch.reset_state.reset = false; - } - - /* Reset arch_timer context */ - return kvm_timer_vcpu_reset(vcpu); -} diff --git a/arch/arm/kvm/trace.h b/arch/arm/kvm/trace.h deleted file mode 100644 index 69a9d62a0ac6..000000000000 --- a/arch/arm/kvm/trace.h +++ /dev/null @@ -1,86 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#if !defined(_TRACE_ARM_KVM_H) || defined(TRACE_HEADER_MULTI_READ) -#define _TRACE_ARM_KVM_H - -#include - -#undef TRACE_SYSTEM -#define TRACE_SYSTEM kvm - -/* Architecturally implementation defined CP15 register access */ -TRACE_EVENT(kvm_emulate_cp15_imp, - TP_PROTO(unsigned long Op1, unsigned long Rt1, unsigned long CRn, - unsigned long CRm, unsigned long Op2, bool is_write), - TP_ARGS(Op1, Rt1, CRn, CRm, Op2, is_write), - - TP_STRUCT__entry( - __field( unsigned int, Op1 ) - __field( unsigned int, Rt1 ) - __field( unsigned int, CRn ) - __field( unsigned int, CRm ) - __field( unsigned int, Op2 ) - __field( bool, is_write ) - ), - - TP_fast_assign( - __entry->is_write = is_write; - __entry->Op1 = Op1; - __entry->Rt1 = Rt1; - __entry->CRn = CRn; - __entry->CRm = CRm; - __entry->Op2 = Op2; - ), - - TP_printk("Implementation defined CP15: %s\tp15, %u, r%u, c%u, c%u, %u", - (__entry->is_write) ? "mcr" : "mrc", - __entry->Op1, __entry->Rt1, __entry->CRn, - __entry->CRm, __entry->Op2) -); - -TRACE_EVENT(kvm_wfx, - TP_PROTO(unsigned long vcpu_pc, bool is_wfe), - TP_ARGS(vcpu_pc, is_wfe), - - TP_STRUCT__entry( - __field( unsigned long, vcpu_pc ) - __field( bool, is_wfe ) - ), - - TP_fast_assign( - __entry->vcpu_pc = vcpu_pc; - __entry->is_wfe = is_wfe; - ), - - TP_printk("guest executed wf%c at: 0x%08lx", - __entry->is_wfe ? 'e' : 'i', __entry->vcpu_pc) -); - -TRACE_EVENT(kvm_hvc, - TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm), - TP_ARGS(vcpu_pc, r0, imm), - - TP_STRUCT__entry( - __field( unsigned long, vcpu_pc ) - __field( unsigned long, r0 ) - __field( unsigned long, imm ) - ), - - TP_fast_assign( - __entry->vcpu_pc = vcpu_pc; - __entry->r0 = r0; - __entry->imm = imm; - ), - - TP_printk("HVC at 0x%08lx (r0: 0x%08lx, imm: 0x%lx", - __entry->vcpu_pc, __entry->r0, __entry->imm) -); - -#endif /* _TRACE_ARM_KVM_H */ - -#undef TRACE_INCLUDE_PATH -#define TRACE_INCLUDE_PATH . -#undef TRACE_INCLUDE_FILE -#define TRACE_INCLUDE_FILE trace - -/* This part must be outside protection */ -#include diff --git a/arch/arm/kvm/vgic-v3-coproc.c b/arch/arm/kvm/vgic-v3-coproc.c deleted file mode 100644 index ed3b2e4759ce..000000000000 --- a/arch/arm/kvm/vgic-v3-coproc.c +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * VGIC system registers handling functions for AArch32 mode - */ - -#include -#include -#include -#include "vgic.h" - -int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, bool is_write, u64 id, - u64 *reg) -{ - /* - * TODO: Implement for AArch32 - */ - return -ENXIO; -} - -int vgic_v3_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, bool is_write, u64 id, - u64 *reg) -{ - /* - * TODO: Implement for AArch32 - */ - return -ENXIO; -} diff --git a/arch/arm/mach-exynos/Kconfig b/arch/arm/mach-exynos/Kconfig index cbbe03e96de8..76838255b5fa 100644 --- a/arch/arm/mach-exynos/Kconfig +++ b/arch/arm/mach-exynos/Kconfig @@ -21,7 +21,7 @@ menuconfig ARCH_EXYNOS select EXYNOS_SROM select EXYNOS_PM_DOMAINS if PM_GENERIC_DOMAINS select GPIOLIB - select HAVE_ARM_ARCH_TIMER if ARCH_EXYNOS5 && VIRTUALIZATION + select HAVE_ARM_ARCH_TIMER if ARCH_EXYNOS5 select HAVE_ARM_SCU if SMP select HAVE_S3C2410_I2C if I2C select HAVE_S3C2410_WATCHDOG if WATCHDOG diff --git a/arch/arm/mach-omap1/board-ams-delta.c b/arch/arm/mach-omap1/board-ams-delta.c index a2aa7a12b374..8d32894ecd2e 100644 --- a/arch/arm/mach-omap1/board-ams-delta.c +++ b/arch/arm/mach-omap1/board-ams-delta.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -294,9 +296,42 @@ struct modem_private_data { static struct modem_private_data modem_priv; +/* + * Define partitions for flash device + */ + +static struct mtd_partition partition_info[] = { + { .name = "Kernel", + .offset = 0, + .size = 3 * SZ_1M + SZ_512K }, + { .name = "u-boot", + .offset = 3 * SZ_1M + SZ_512K, + .size = SZ_256K }, + { .name = "u-boot params", + .offset = 3 * SZ_1M + SZ_512K + SZ_256K, + .size = SZ_256K }, + { .name = "Amstrad LDR", + .offset = 4 * SZ_1M, + .size = SZ_256K }, + { .name = "File system", + .offset = 4 * SZ_1M + 1 * SZ_256K, + .size = 27 * SZ_1M }, + { .name = "PBL reserved", + .offset = 32 * SZ_1M - 3 * SZ_256K, + .size = 3 * SZ_256K }, +}; + +static struct gpio_nand_platdata nand_platdata = { + .parts = partition_info, + .num_parts = ARRAY_SIZE(partition_info), +}; + static struct platform_device ams_delta_nand_device = { .name = "ams-delta-nand", .id = -1, + .dev = { + .platform_data = &nand_platdata, + }, }; #define OMAP_GPIO_LABEL "gpio-0-15" @@ -306,10 +341,14 @@ static struct gpiod_lookup_table ams_delta_nand_gpio_table = { .table = { GPIO_LOOKUP(OMAP_GPIO_LABEL, AMS_DELTA_GPIO_PIN_NAND_RB, "rdy", 0), - GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NCE, "nce", 0), - GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NRE, "nre", 0), - GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NWP, "nwp", 0), - GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NWE, "nwe", 0), + GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NCE, "nce", + GPIO_ACTIVE_LOW), + GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NRE, "nre", + GPIO_ACTIVE_LOW), + GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NWP, "nwp", + GPIO_ACTIVE_LOW), + GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_NWE, "nwe", + GPIO_ACTIVE_LOW), GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_ALE, "ale", 0), GPIO_LOOKUP(LATCH2_LABEL, LATCH2_PIN_NAND_CLE, "cle", 0), GPIO_LOOKUP_IDX(OMAP_MPUIO_LABEL, 0, "data", 0, 0), diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index bd0f4821f7e1..b598e6978b29 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -241,7 +241,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) struct mm_struct *mm; int sig, code; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; if (kprobe_page_fault(regs, fsr)) return 0; @@ -295,7 +295,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * signal first. We do not need to release the mmap_sem because * it would already be released in __lock_page_or_retry in * mm/filemap.c. */ - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { + if (fault_signal_pending(fault, regs)) { if (!user_mode(regs)) goto no_context; return 0; @@ -319,9 +319,6 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) regs, addr); } if (fault & VM_FAULT_RETRY) { - /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk - * of starvation. */ - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; goto retry; } diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 5d0d0f86e790..69a337df619f 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -63,9 +63,6 @@ static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK; static unsigned int ecc_mask __initdata = 0; pgprot_t pgprot_user; pgprot_t pgprot_kernel; -pgprot_t pgprot_hyp_device; -pgprot_t pgprot_s2; -pgprot_t pgprot_s2_device; EXPORT_SYMBOL(pgprot_user); EXPORT_SYMBOL(pgprot_kernel); @@ -75,15 +72,8 @@ struct cachepolicy { unsigned int cr_mask; pmdval_t pmd; pteval_t pte; - pteval_t pte_s2; }; -#ifdef CONFIG_ARM_LPAE -#define s2_policy(policy) policy -#else -#define s2_policy(policy) 0 -#endif - unsigned long kimage_voffset __ro_after_init; static struct cachepolicy cache_policies[] __initdata = { @@ -92,31 +82,26 @@ static struct cachepolicy cache_policies[] __initdata = { .cr_mask = CR_W|CR_C, .pmd = PMD_SECT_UNCACHED, .pte = L_PTE_MT_UNCACHED, - .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED), }, { .policy = "buffered", .cr_mask = CR_C, .pmd = PMD_SECT_BUFFERED, .pte = L_PTE_MT_BUFFERABLE, - .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED), }, { .policy = "writethrough", .cr_mask = 0, .pmd = PMD_SECT_WT, .pte = L_PTE_MT_WRITETHROUGH, - .pte_s2 = s2_policy(L_PTE_S2_MT_WRITETHROUGH), }, { .policy = "writeback", .cr_mask = 0, .pmd = PMD_SECT_WB, .pte = L_PTE_MT_WRITEBACK, - .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK), }, { .policy = "writealloc", .cr_mask = 0, .pmd = PMD_SECT_WBWA, .pte = L_PTE_MT_WRITEALLOC, - .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK), } }; @@ -246,9 +231,6 @@ static struct mem_type mem_types[] __ro_after_init = { [MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */ .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED | L_PTE_SHARED, - .prot_pte_s2 = s2_policy(PROT_PTE_S2_DEVICE) | - s2_policy(L_PTE_S2_MT_DEV_SHARED) | - L_PTE_SHARED, .prot_l1 = PMD_TYPE_TABLE, .prot_sect = PROT_SECT_DEVICE | PMD_SECT_S, .domain = DOMAIN_IO, @@ -434,7 +416,6 @@ static void __init build_mem_type_table(void) struct cachepolicy *cp; unsigned int cr = get_cr(); pteval_t user_pgprot, kern_pgprot, vecs_pgprot; - pteval_t hyp_device_pgprot, s2_pgprot, s2_device_pgprot; int cpu_arch = cpu_architecture(); int i; @@ -558,9 +539,6 @@ static void __init build_mem_type_table(void) */ cp = &cache_policies[cachepolicy]; vecs_pgprot = kern_pgprot = user_pgprot = cp->pte; - s2_pgprot = cp->pte_s2; - hyp_device_pgprot = mem_types[MT_DEVICE].prot_pte; - s2_device_pgprot = mem_types[MT_DEVICE].prot_pte_s2; #ifndef CONFIG_ARM_LPAE /* @@ -604,7 +582,6 @@ static void __init build_mem_type_table(void) user_pgprot |= L_PTE_SHARED; kern_pgprot |= L_PTE_SHARED; vecs_pgprot |= L_PTE_SHARED; - s2_pgprot |= L_PTE_SHARED; mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S; mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED; mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S; @@ -666,9 +643,6 @@ static void __init build_mem_type_table(void) pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot); pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | kern_pgprot); - pgprot_s2 = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | s2_pgprot); - pgprot_s2_device = __pgprot(s2_device_pgprot); - pgprot_hyp_device = __pgprot(hyp_device_pgprot); mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask; mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index d3077c991962..ff9cbb631212 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild @@ -1,26 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 -generic-y += bugs.h -generic-y += delay.h -generic-y += div64.h -generic-y += dma.h -generic-y += dma-mapping.h generic-y += early_ioremap.h -generic-y += emergency-restart.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h generic-y += qrwlock.h generic-y += qspinlock.h -generic-y += serial.h generic-y += set_memory.h -generic-y += switch_to.h -generic-y += trace_clock.h -generic-y += unaligned.h generic-y += user.h -generic-y += vga.h diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index f658dda12364..a30b4eec7cb4 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -89,7 +89,8 @@ static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) static inline void vcpu_clear_wfx_traps(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 &= ~HCR_TWE; - if (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count)) + if (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) || + vcpu->kvm->arch.vgic.nassgireq) vcpu->arch.hcr_el2 &= ~HCR_TWI; else vcpu->arch.hcr_el2 |= HCR_TWI; diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 57fd46acd058..32c8a675e5a4 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -44,6 +44,7 @@ #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) #define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2) #define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(3) +#define KVM_REQ_RELOAD_GICv4 KVM_ARCH_REQ(4) DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use); diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index 525010504f9d..e329a36b2bee 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 2bd92301d32f..23ebe51410f0 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include "trace.h" diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c index eaa05c3c7235..8a1e81a400e0 100644 --- a/arch/arm64/kvm/hyp/switch.c +++ b/arch/arm64/kvm/hyp/switch.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 090f46d3add1..51db934702b6 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/arm64/kvm/sys_regs_generic_v8.c b/arch/arm64/kvm/sys_regs_generic_v8.c index 2b4a3e2d1b89..9cb6b4c8355a 100644 --- a/arch/arm64/kvm/sys_regs_generic_v8.c +++ b/arch/arm64/kvm/sys_regs_generic_v8.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 85566d32958f..1027851d469a 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -446,7 +446,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, struct mm_struct *mm = current->mm; vm_fault_t fault, major = 0; unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC; - unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int mm_flags = FAULT_FLAG_DEFAULT; if (kprobe_page_fault(regs, esr)) return 0; @@ -513,25 +513,15 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, fault = __do_page_fault(mm, addr, mm_flags, vm_flags); major |= fault & VM_FAULT_MAJOR; - if (fault & VM_FAULT_RETRY) { - /* - * If we need to retry but a fatal signal is pending, - * handle the signal first. We do not need to release - * the mmap_sem because it would already be released - * in __lock_page_or_retry in mm/filemap.c. - */ - if (fatal_signal_pending(current)) { - if (!user_mode(regs)) - goto no_context; - return 0; - } + /* Quick path to respond to signals */ + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + goto no_context; + return 0; + } - /* - * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of - * starvation. - */ + if (fault & VM_FAULT_RETRY) { if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { - mm_flags &= ~FAULT_FLAG_ALLOW_RETRY; mm_flags |= FAULT_FLAG_TRIED; goto retry; } diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild index a036e05fc3c6..a4ef93a1f7ae 100644 --- a/arch/c6x/include/asm/Kbuild +++ b/arch/c6x/include/asm/Kbuild @@ -1,42 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -generic-y += atomic.h -generic-y += barrier.h -generic-y += bugs.h -generic-y += compat.h -generic-y += current.h -generic-y += device.h -generic-y += div64.h -generic-y += dma.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += extable.h -generic-y += fb.h -generic-y += futex.h -generic-y += hw_irq.h -generic-y += io.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += local.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += mmu.h -generic-y += mmu_context.h -generic-y += pci.h -generic-y += percpu.h -generic-y += pgalloc.h -generic-y += preempt.h -generic-y += serial.h -generic-y += shmparam.h -generic-y += tlbflush.h -generic-y += topology.h -generic-y += trace_clock.h generic-y += user.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/csky/include/asm/Kbuild b/arch/csky/include/asm/Kbuild index 4130e3eaa766..93372255984d 100644 --- a/arch/csky/include/asm/Kbuild +++ b/arch/csky/include/asm/Kbuild @@ -1,44 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 generic-y += asm-offsets.h -generic-y += bugs.h -generic-y += compat.h -generic-y += current.h -generic-y += delay.h -generic-y += device.h -generic-y += div64.h -generic-y += dma.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h -generic-y += fb.h -generic-y += futex.h generic-y += gpio.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += linkage.h -generic-y += local.h generic-y += local64.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += module.h -generic-y += percpu.h -generic-y += preempt.h generic-y += qrwlock.h -generic-y += sections.h -generic-y += serial.h -generic-y += timex.h -generic-y += topology.h -generic-y += trace_clock.h -generic-y += unaligned.h generic-y += user.h -generic-y += vga.h generic-y += vmlinux.lds.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/h8300/include/asm/Kbuild b/arch/h8300/include/asm/Kbuild index 7f618190e1a9..ddf04f32b546 100644 --- a/arch/h8300/include/asm/Kbuild +++ b/arch/h8300/include/asm/Kbuild @@ -1,54 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 generic-y += asm-offsets.h -generic-y += barrier.h -generic-y += bugs.h -generic-y += cacheflush.h -generic-y += checksum.h -generic-y += compat.h -generic-y += current.h -generic-y += delay.h -generic-y += device.h -generic-y += div64.h -generic-y += dma.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += extable.h -generic-y += fb.h -generic-y += ftrace.h -generic-y += futex.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += linkage.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += mmu.h -generic-y += mmu_context.h -generic-y += module.h generic-y += parport.h -generic-y += pci.h -generic-y += percpu.h -generic-y += pgalloc.h -generic-y += preempt.h -generic-y += sections.h -generic-y += serial.h -generic-y += shmparam.h generic-y += spinlock.h -generic-y += timex.h -generic-y += tlbflush.h -generic-y += topology.h -generic-y += trace_clock.h -generic-y += uaccess.h -generic-y += unaligned.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild index 84bb1ed1b931..373964bb177e 100644 --- a/arch/hexagon/include/asm/Kbuild +++ b/arch/hexagon/include/asm/Kbuild @@ -1,39 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -generic-y += barrier.h -generic-y += bug.h -generic-y += bugs.h -generic-y += compat.h -generic-y += current.h -generic-y += device.h -generic-y += div64.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h generic-y += extable.h -generic-y += fb.h -generic-y += ftrace.h -generic-y += hardirq.h -generic-y += hw_irq.h generic-y += iomap.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += pci.h -generic-y += percpu.h -generic-y += preempt.h -generic-y += sections.h -generic-y += serial.h -generic-y += shmparam.h -generic-y += topology.h -generic-y += trace_clock.h -generic-y += unaligned.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/hexagon/mm/vm_fault.c b/arch/hexagon/mm/vm_fault.c index b3bc71680ae4..72334b26317a 100644 --- a/arch/hexagon/mm/vm_fault.c +++ b/arch/hexagon/mm/vm_fault.c @@ -41,7 +41,7 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs) int si_code = SEGV_MAPERR; vm_fault_t fault; const struct exception_table_entry *fixup; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; /* * If we're in an interrupt or have no user context, @@ -91,7 +91,7 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs) fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; /* The most common case -- we are done. */ @@ -102,7 +102,6 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs) else current->min_flt++; if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; goto retry; } diff --git a/arch/ia64/include/asm/Kbuild b/arch/ia64/include/asm/Kbuild index 390393667d3b..f994c1daf9d4 100644 --- a/arch/ia64/include/asm/Kbuild +++ b/arch/ia64/include/asm/Kbuild @@ -1,12 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += syscall_table.h -generic-y += compat.h -generic-y += exec.h -generic-y += irq_work.h generic-y += kvm_para.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += preempt.h -generic-y += trace_clock.h generic-y += vtime.h -generic-y += word-at-a-time.h diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index c2f299fe9e04..30d0c1fca99e 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -65,7 +65,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re struct mm_struct *mm = current->mm; unsigned long mask; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; mask = ((((isr >> IA64_ISR_X_BIT) & 1UL) << VM_EXEC_BIT) | (((isr >> IA64_ISR_W_BIT) & 1UL) << VM_WRITE_BIT)); @@ -141,7 +141,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re */ fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -167,7 +167,6 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re else current->min_flt++; if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* No need to up_read(&mm->mmap_sem) as we would diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild index 63f12afc4874..a0765aa60ea9 100644 --- a/arch/m68k/include/asm/Kbuild +++ b/arch/m68k/include/asm/Kbuild @@ -1,32 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += syscall_table.h -generic-y += barrier.h -generic-y += compat.h -generic-y += device.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += extable.h -generic-y += futex.h generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += percpu.h -generic-y += preempt.h -generic-y += sections.h -generic-y += shmparam.h generic-y += spinlock.h -generic-y += topology.h -generic-y += trace_clock.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index e9b1d7585b43..f7afb9897966 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -71,7 +71,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, struct mm_struct *mm = current->mm; struct vm_area_struct * vma; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; pr_debug("do page fault:\nregs->sr=%#x, regs->pc=%#lx, address=%#lx, %ld, %p\n", regs->sr, regs->pc, address, error_code, mm ? mm->pgd : NULL); @@ -138,7 +138,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, fault = handle_mm_fault(vma, address, flags); pr_debug("handle_mm_fault returns %x\n", fault); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return 0; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -162,9 +162,6 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, else current->min_flt++; if (fault & VM_FAULT_RETRY) { - /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk - * of starvation. */ - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild index abb33619299b..2e87a9b6d312 100644 --- a/arch/microblaze/include/asm/Kbuild +++ b/arch/microblaze/include/asm/Kbuild @@ -1,40 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += syscall_table.h -generic-y += bitops.h -generic-y += bug.h -generic-y += bugs.h -generic-y += compat.h -generic-y += device.h -generic-y += div64.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += extable.h -generic-y += fb.h -generic-y += hardirq.h generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += linkage.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h generic-y += parport.h -generic-y += percpu.h -generic-y += preempt.h -generic-y += serial.h -generic-y += shmparam.h generic-y += syscalls.h generic-y += tlb.h -generic-y += topology.h -generic-y += trace_clock.h generic-y += user.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index e6a810b0c7ad..3248141f8ed5 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c @@ -91,7 +91,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, int code = SEGV_MAPERR; int is_write = error_code & ESR_S; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; regs->ear = address; regs->esr = error_code; @@ -217,7 +217,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, */ fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -236,7 +236,6 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, else current->min_flt++; if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* diff --git a/arch/mips/boot/dts/brcm/bcm7425.dtsi b/arch/mips/boot/dts/brcm/bcm7425.dtsi index 410e61ebaf9e..aa0b2d39c902 100644 --- a/arch/mips/boot/dts/brcm/bcm7425.dtsi +++ b/arch/mips/boot/dts/brcm/bcm7425.dtsi @@ -403,8 +403,8 @@ nand: nand@41b800 { compatible = "brcm,brcmnand-v5.0", "brcm,brcmnand"; #address-cells = <1>; #size-cells = <0>; - reg-names = "nand"; - reg = <0x41b800 0x400>; + reg-names = "nand", "flash-edu"; + reg = <0x41b800 0x400>, <0x41bc00 0x24>; interrupt-parent = <&hif_l2_intc>; interrupts = <24>; status = "disabled"; diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild index 4ebd8ce254ce..8643d313890e 100644 --- a/arch/mips/include/asm/Kbuild +++ b/arch/mips/include/asm/Kbuild @@ -4,23 +4,10 @@ generated-y += syscall_table_32_o32.h generated-y += syscall_table_64_n32.h generated-y += syscall_table_64_n64.h generated-y += syscall_table_64_o32.h -generic-y += current.h -generic-y += device.h -generic-y += emergency-restart.h generic-y += export.h -generic-y += irq_work.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h generic-y += parport.h -generic-y += percpu.h -generic-y += preempt.h generic-y += qrwlock.h generic-y += qspinlock.h -generic-y += sections.h -generic-y += serial.h -generic-y += trace_clock.h -generic-y += unaligned.h generic-y += user.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index 41204a49cf95..2c343c346b79 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -1133,7 +1133,7 @@ extern unsigned long kvm_mips_get_ramsize(struct kvm *kvm); static inline void kvm_arch_hardware_unsetup(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, - struct kvm_memory_slot *free, struct kvm_memory_slot *dont) {} + struct kvm_memory_slot *slot) {} static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 71244bf87c3a..8f05dd0a0f4e 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -118,12 +118,12 @@ void kvm_arch_hardware_disable(void) kvm_mips_callbacks->hardware_disable(); } -int kvm_arch_hardware_setup(void) +int kvm_arch_hardware_setup(void *opaque) { return 0; } -int kvm_arch_check_processor_compat(void) +int kvm_arch_check_processor_compat(void *opaque) { return 0; } @@ -188,12 +188,6 @@ long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, return -ENOIOCTLCMD; } -int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned long npages) -{ - return 0; -} - void kvm_arch_flush_shadow_all(struct kvm *kvm) { /* Flush whole GPA */ @@ -230,7 +224,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { @@ -984,69 +978,16 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, return r; } -/** - * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot - * @kvm: kvm instance - * @log: slot id and address to which we copy the log - * - * Steps 1-4 below provide general overview of dirty page logging. See - * kvm_get_dirty_log_protect() function description for additional details. - * - * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we - * always flush the TLB (step 4) even if previous step failed and the dirty - * bitmap may be corrupt. Regardless of previous outcome the KVM logging API - * does not preclude user space subsequent dirty log read. Flushing TLB ensures - * writes will be marked dirty for next log read. - * - * 1. Take a snapshot of the bit and clear it if needed. - * 2. Write protect the corresponding page. - * 3. Copy the snapshot to the userspace. - * 4. Flush TLB's if needed. - */ -int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { - struct kvm_memslots *slots; - struct kvm_memory_slot *memslot; - bool flush = false; - int r; - mutex_lock(&kvm->slots_lock); - - r = kvm_get_dirty_log_protect(kvm, log, &flush); - - if (flush) { - slots = kvm_memslots(kvm); - memslot = id_to_memslot(slots, log->slot); - - /* Let implementation handle TLB/GVA invalidation */ - kvm_mips_callbacks->flush_shadow_memslot(kvm, memslot); - } - - mutex_unlock(&kvm->slots_lock); - return r; } -int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log) +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) { - struct kvm_memslots *slots; - struct kvm_memory_slot *memslot; - bool flush = false; - int r; - - mutex_lock(&kvm->slots_lock); - - r = kvm_clear_dirty_log_protect(kvm, log, &flush); - - if (flush) { - slots = kvm_memslots(kvm); - memslot = id_to_memslot(slots, log->slot); - - /* Let implementation handle TLB/GVA invalidation */ - kvm_mips_callbacks->flush_shadow_memslot(kvm, memslot); - } - - mutex_unlock(&kvm->slots_lock); - return r; + /* Let implementation handle TLB/GVA invalidation */ + kvm_mips_callbacks->flush_shadow_memslot(kvm, memslot); } long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 1e8d00793784..4a0eafe3d932 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -44,7 +44,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, const int field = sizeof(unsigned long) * 2; int si_code; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); @@ -154,7 +154,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, */ fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); @@ -178,7 +178,6 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, tsk->min_flt++; } if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* diff --git a/arch/nds32/include/asm/Kbuild b/arch/nds32/include/asm/Kbuild index 77eae62036b5..ff1e94299317 100644 --- a/arch/nds32/include/asm/Kbuild +++ b/arch/nds32/include/asm/Kbuild @@ -1,46 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 generic-y += asm-offsets.h -generic-y += atomic.h -generic-y += bitops.h -generic-y += bug.h -generic-y += bugs.h -generic-y += checksum.h generic-y += cmpxchg.h -generic-y += compat.h -generic-y += device.h -generic-y += div64.h -generic-y += dma.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += export.h -generic-y += fb.h generic-y += gpio.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += local.h generic-y += local64.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h generic-y += parport.h -generic-y += pci.h -generic-y += percpu.h -generic-y += preempt.h -generic-y += sections.h -generic-y += serial.h -generic-y += switch_to.h -generic-y += timex.h -generic-y += topology.h -generic-y += trace_clock.h -generic-y += xor.h -generic-y += unaligned.h generic-y += user.h -generic-y += vga.h -generic-y += word-at-a-time.h diff --git a/arch/nds32/mm/fault.c b/arch/nds32/mm/fault.c index 906dfb25353c..0cf0c08c7da2 100644 --- a/arch/nds32/mm/fault.c +++ b/arch/nds32/mm/fault.c @@ -80,7 +80,7 @@ void do_page_fault(unsigned long entry, unsigned long addr, int si_code; vm_fault_t fault; unsigned int mask = VM_READ | VM_WRITE | VM_EXEC; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; error_code = error_code & (ITYPE_mskINST | ITYPE_mskETYPE); tsk = current; @@ -214,7 +214,7 @@ void do_page_fault(unsigned long entry, unsigned long addr, * signal first. We do not need to release the mmap_sem because it * would already be released in __lock_page_or_retry in mm/filemap.c. */ - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { + if (fault_signal_pending(fault, regs)) { if (!user_mode(regs)) goto no_context; return; @@ -246,7 +246,6 @@ void do_page_fault(unsigned long entry, unsigned long addr, 1, regs, addr); } if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* No need to up_read(&mm->mmap_sem) as we would diff --git a/arch/nios2/include/asm/Kbuild b/arch/nios2/include/asm/Kbuild index 68093999bd26..7fe7437555fb 100644 --- a/arch/nios2/include/asm/Kbuild +++ b/arch/nios2/include/asm/Kbuild @@ -1,45 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -generic-y += atomic.h -generic-y += barrier.h -generic-y += bitops.h -generic-y += bug.h -generic-y += bugs.h generic-y += cmpxchg.h -generic-y += compat.h -generic-y += current.h -generic-y += device.h -generic-y += div64.h -generic-y += dma.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += extable.h -generic-y += fb.h -generic-y += ftrace.h -generic-y += futex.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += local.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += module.h -generic-y += pci.h -generic-y += percpu.h -generic-y += preempt.h -generic-y += sections.h -generic-y += serial.h generic-y += spinlock.h -generic-y += topology.h -generic-y += trace_clock.h -generic-y += unaligned.h generic-y += user.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c index 6a2e716b959f..ec9d8a9c426f 100644 --- a/arch/nios2/mm/fault.c +++ b/arch/nios2/mm/fault.c @@ -47,7 +47,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, struct mm_struct *mm = tsk->mm; int code = SEGV_MAPERR; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; cause >>= 2; @@ -133,7 +133,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, */ fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -157,9 +157,6 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, else current->min_flt++; if (fault & VM_FAULT_RETRY) { - /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk - * of starvation. */ - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild index e12d6c1735a0..ca5987e11053 100644 --- a/arch/openrisc/include/asm/Kbuild +++ b/arch/openrisc/include/asm/Kbuild @@ -1,45 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 -generic-y += barrier.h -generic-y += bug.h -generic-y += bugs.h -generic-y += checksum.h -generic-y += compat.h -generic-y += current.h -generic-y += device.h -generic-y += div64.h -generic-y += dma.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += extable.h -generic-y += fb.h -generic-y += ftrace.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += local.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += module.h -generic-y += pci.h -generic-y += percpu.h -generic-y += preempt.h generic-y += qspinlock_types.h generic-y += qspinlock.h generic-y += qrwlock_types.h generic-y += qrwlock.h -generic-y += sections.h -generic-y += shmparam.h -generic-y += switch_to.h -generic-y += topology.h -generic-y += trace_clock.h generic-y += user.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/openrisc/mm/fault.c b/arch/openrisc/mm/fault.c index 5d4d3a9691d0..8af1cc78c4fb 100644 --- a/arch/openrisc/mm/fault.c +++ b/arch/openrisc/mm/fault.c @@ -50,7 +50,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address, struct vm_area_struct *vma; int si_code; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; tsk = current; @@ -161,7 +161,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address, fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -181,7 +181,6 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address, else tsk->min_flt++; if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* No need to up_read(&mm->mmap_sem) as we would diff --git a/arch/parisc/include/asm/Kbuild b/arch/parisc/include/asm/Kbuild index 9ceedf6393c4..e3ee5c0bfe80 100644 --- a/arch/parisc/include/asm/Kbuild +++ b/arch/parisc/include/asm/Kbuild @@ -2,26 +2,8 @@ generated-y += syscall_table_32.h generated-y += syscall_table_64.h generated-y += syscall_table_c32.h -generic-y += current.h -generic-y += device.h -generic-y += div64.h -generic-y += emergency-restart.h -generic-y += exec.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h generic-y += kvm_para.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += percpu.h -generic-y += preempt.h generic-y += seccomp.h -generic-y += trace_clock.h generic-y += user.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index adbd5e2144a3..86e8c848f3d7 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -274,7 +274,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, if (!mm) goto no_context; - flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + flags = FAULT_FLAG_DEFAULT; if (user_mode(regs)) flags |= FAULT_FLAG_USER; @@ -304,7 +304,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -328,14 +328,12 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, else current->min_flt++; if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; - /* * No need to up_read(&mm->mmap_sem) as we would * have already released it in __lock_page_or_retry * in mm/filemap.c. */ - + flags |= FAULT_FLAG_TRIED; goto retry; } } diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 6f40af294685..5fc45364e86e 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -980,6 +980,7 @@ config PPC_SECURE_BOOT bool depends on PPC_POWERNV depends on IMA_ARCH_POLICY + imply IMA_SECURE_AND_OR_TRUSTED_BOOT help Systems with firmware secure boot enabled need to define security policies to extend secure boot to the OS. This config allows a user diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild index d0a23d0db863..dadbcf3a0b1e 100644 --- a/arch/powerpc/include/asm/Kbuild +++ b/arch/powerpc/include/asm/Kbuild @@ -3,12 +3,8 @@ generated-y += syscall_table_32.h generated-y += syscall_table_64.h generated-y += syscall_table_c32.h generated-y += syscall_table_spu.h -generic-y += div64.h -generic-y += dma-mapping.h generic-y += export.h -generic-y += irq_regs.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += preempt.h generic-y += vtime.h generic-y += early_ioremap.h diff --git a/arch/powerpc/include/asm/kvm_asm.h b/arch/powerpc/include/asm/kvm_asm.h index 635fb154b33f..a3633560493b 100644 --- a/arch/powerpc/include/asm/kvm_asm.h +++ b/arch/powerpc/include/asm/kvm_asm.h @@ -150,4 +150,7 @@ #define KVM_INST_FETCH_FAILED -1 +/* Extract PO and XOP opcode fields */ +#define PO_XOP_OPCODE_MASK 0xfc0007fe + #endif /* __POWERPC_KVM_ASM_H__ */ diff --git a/arch/powerpc/include/asm/kvm_book3s_uvmem.h b/arch/powerpc/include/asm/kvm_book3s_uvmem.h index 5a9834e0e2d1..9cb7d8be2366 100644 --- a/arch/powerpc/include/asm/kvm_book3s_uvmem.h +++ b/arch/powerpc/include/asm/kvm_book3s_uvmem.h @@ -5,6 +5,7 @@ #ifdef CONFIG_PPC_UV int kvmppc_uvmem_init(void); void kvmppc_uvmem_free(void); +bool kvmppc_uvmem_available(void); int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot); void kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot); @@ -30,6 +31,11 @@ static inline int kvmppc_uvmem_init(void) static inline void kvmppc_uvmem_free(void) { } +static inline bool kvmppc_uvmem_available(void) +{ + return false; +} + static inline int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot) { diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 6e8b8ffd06ad..f99b4333dfba 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -303,6 +303,7 @@ struct kvm_arch { u8 radix; u8 fwnmi_enabled; u8 secure_guest; + u8 svm_enabled; bool threads_indep; bool nested_enable; pgd_t *pgtable; diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index bc2494e5710a..94f5a32acaf1 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -107,8 +107,6 @@ extern void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gpa_t gpaddr, unsigned int gtlb_idx); extern void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode); extern void kvmppc_mmu_switch_pid(struct kvm_vcpu *vcpu, u32 pid); -extern void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu); -extern int kvmppc_mmu_init(struct kvm_vcpu *vcpu); extern int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr); extern int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr); extern gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int gtlb_index, @@ -200,14 +198,11 @@ extern void kvm_free_hpt_cma(struct page *page, unsigned long nr_pages); extern int kvmppc_core_init_vm(struct kvm *kvm); extern void kvmppc_core_destroy_vm(struct kvm *kvm); extern void kvmppc_core_free_memslot(struct kvm *kvm, - struct kvm_memory_slot *free, - struct kvm_memory_slot *dont); -extern int kvmppc_core_create_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot, - unsigned long npages); + struct kvm_memory_slot *slot); extern int kvmppc_core_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem); + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change); extern void kvmppc_core_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, const struct kvm_memory_slot *old, @@ -280,7 +275,8 @@ struct kvmppc_ops { void (*flush_memslot)(struct kvm *kvm, struct kvm_memory_slot *memslot); int (*prepare_memory_region)(struct kvm *kvm, struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem); + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change); void (*commit_memory_region)(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, const struct kvm_memory_slot *old, @@ -291,11 +287,7 @@ struct kvmppc_ops { int (*age_hva)(struct kvm *kvm, unsigned long start, unsigned long end); int (*test_age_hva)(struct kvm *kvm, unsigned long hva); void (*set_spte_hva)(struct kvm *kvm, unsigned long hva, pte_t pte); - void (*mmu_destroy)(struct kvm_vcpu *vcpu); - void (*free_memslot)(struct kvm_memory_slot *free, - struct kvm_memory_slot *dont); - int (*create_memslot)(struct kvm_memory_slot *slot, - unsigned long npages); + void (*free_memslot)(struct kvm_memory_slot *slot); int (*init_vm)(struct kvm *kvm); void (*destroy_vm)(struct kvm *kvm); int (*get_smmu_info)(struct kvm *kvm, struct kvm_ppc_smmu_info *info); @@ -321,6 +313,7 @@ struct kvmppc_ops { int size); int (*store_to_eaddr)(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr, int size); + int (*enable_svm)(struct kvm *kvm); int (*svm_off)(struct kvm *kvm); }; diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index d07a8e12fa15..5690a1f9b976 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -799,21 +799,19 @@ int kvmppc_core_check_requests(struct kvm_vcpu *vcpu) return vcpu->kvm->arch.kvm_ops->check_requests(vcpu); } +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + +} + int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) { return kvm->arch.kvm_ops->get_dirty_log(kvm, log); } -void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) +void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { - kvm->arch.kvm_ops->free_memslot(free, dont); -} - -int kvmppc_core_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned long npages) -{ - return kvm->arch.kvm_ops->create_memslot(slot, npages); + kvm->arch.kvm_ops->free_memslot(slot); } void kvmppc_core_flush_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot) @@ -823,9 +821,11 @@ void kvmppc_core_flush_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot) int kvmppc_core_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem) + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) { - return kvm->arch.kvm_ops->prepare_memory_region(kvm, memslot, mem); + return kvm->arch.kvm_ops->prepare_memory_region(kvm, memslot, mem, + change); } void kvmppc_core_commit_memory_region(struct kvm *kvm, @@ -858,11 +858,6 @@ int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) return 0; } -void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu) -{ - vcpu->kvm->arch.kvm_ops->mmu_destroy(vcpu); -} - int kvmppc_core_init_vm(struct kvm *kvm) { diff --git a/arch/powerpc/kvm/book3s.h b/arch/powerpc/kvm/book3s.h index 3a4613985949..eae259ee49af 100644 --- a/arch/powerpc/kvm/book3s.h +++ b/arch/powerpc/kvm/book3s.h @@ -16,6 +16,7 @@ extern int kvm_age_hva_hv(struct kvm *kvm, unsigned long start, extern int kvm_test_age_hva_hv(struct kvm *kvm, unsigned long hva); extern void kvm_set_spte_hva_hv(struct kvm *kvm, unsigned long hva, pte_t pte); +extern int kvmppc_mmu_init_pr(struct kvm_vcpu *vcpu); extern void kvmppc_mmu_destroy_pr(struct kvm_vcpu *vcpu); extern int kvmppc_core_emulate_op_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, unsigned int inst, int *advance); diff --git a/arch/powerpc/kvm/book3s_32_mmu.c b/arch/powerpc/kvm/book3s_32_mmu.c index f21e73492ce3..3fbd570f9c1e 100644 --- a/arch/powerpc/kvm/book3s_32_mmu.c +++ b/arch/powerpc/kvm/book3s_32_mmu.c @@ -234,7 +234,7 @@ static int kvmppc_mmu_book3s_32_xlate_pte(struct kvm_vcpu *vcpu, gva_t eaddr, case 2: case 6: pte->may_write = true; - /* fall through */ + fallthrough; case 3: case 5: case 7: diff --git a/arch/powerpc/kvm/book3s_32_mmu_host.c b/arch/powerpc/kvm/book3s_32_mmu_host.c index d4cb3bcf41b6..e8e7b2c530d1 100644 --- a/arch/powerpc/kvm/book3s_32_mmu_host.c +++ b/arch/powerpc/kvm/book3s_32_mmu_host.c @@ -356,7 +356,7 @@ void kvmppc_mmu_destroy_pr(struct kvm_vcpu *vcpu) /* From mm/mmu_context_hash32.c */ #define CTX_TO_VSID(c, id) ((((c) * (897 * 16)) + (id * 0x111)) & 0xffffff) -int kvmppc_mmu_init(struct kvm_vcpu *vcpu) +int kvmppc_mmu_init_pr(struct kvm_vcpu *vcpu) { struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); int err; diff --git a/arch/powerpc/kvm/book3s_64_mmu.c b/arch/powerpc/kvm/book3s_64_mmu.c index 599133256a95..26b8b27a3755 100644 --- a/arch/powerpc/kvm/book3s_64_mmu.c +++ b/arch/powerpc/kvm/book3s_64_mmu.c @@ -311,7 +311,7 @@ static int kvmppc_mmu_book3s_64_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, case 2: case 6: gpte->may_write = true; - /* fall through */ + fallthrough; case 3: case 5: case 7: diff --git a/arch/powerpc/kvm/book3s_64_mmu_host.c b/arch/powerpc/kvm/book3s_64_mmu_host.c index 044dd49eeb9d..e452158a18d7 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_host.c +++ b/arch/powerpc/kvm/book3s_64_mmu_host.c @@ -384,7 +384,7 @@ void kvmppc_mmu_destroy_pr(struct kvm_vcpu *vcpu) __destroy_context(to_book3s(vcpu)->context_id[0]); } -int kvmppc_mmu_init(struct kvm_vcpu *vcpu) +int kvmppc_mmu_init_pr(struct kvm_vcpu *vcpu) { struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); int err; diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index 6c372f5c61b6..3aecec890d6f 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -485,18 +485,18 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, __be64 *hptep; unsigned long mmu_seq, psize, pte_size; unsigned long gpa_base, gfn_base; - unsigned long gpa, gfn, hva, pfn; + unsigned long gpa, gfn, hva, pfn, hpa; struct kvm_memory_slot *memslot; unsigned long *rmap; struct revmap_entry *rev; - struct page *page, *pages[1]; - long index, ret, npages; + struct page *page; + long index, ret; bool is_ci; - unsigned int writing, write_ok; - struct vm_area_struct *vma; + bool writing, write_ok; + unsigned int shift; unsigned long rcbits; long mmio_update; - struct mm_struct *mm; + pte_t pte, *ptep; if (kvm_is_radix(kvm)) return kvmppc_book3s_radix_page_fault(run, vcpu, ea, dsisr); @@ -570,59 +570,62 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, smp_rmb(); ret = -EFAULT; - is_ci = false; - pfn = 0; page = NULL; - mm = kvm->mm; - pte_size = PAGE_SIZE; writing = (dsisr & DSISR_ISSTORE) != 0; /* If writing != 0, then the HPTE must allow writing, if we get here */ write_ok = writing; hva = gfn_to_hva_memslot(memslot, gfn); - npages = get_user_pages_fast(hva, 1, writing ? FOLL_WRITE : 0, pages); - if (npages < 1) { - /* Check if it's an I/O mapping */ - down_read(&mm->mmap_sem); - vma = find_vma(mm, hva); - if (vma && vma->vm_start <= hva && hva + psize <= vma->vm_end && - (vma->vm_flags & VM_PFNMAP)) { - pfn = vma->vm_pgoff + - ((hva - vma->vm_start) >> PAGE_SHIFT); - pte_size = psize; - is_ci = pte_ci(__pte((pgprot_val(vma->vm_page_prot)))); - write_ok = vma->vm_flags & VM_WRITE; - } - up_read(&mm->mmap_sem); - if (!pfn) - goto out_put; + + /* + * Do a fast check first, since __gfn_to_pfn_memslot doesn't + * do it with !atomic && !async, which is how we call it. + * We always ask for write permission since the common case + * is that the page is writable. + */ + if (__get_user_pages_fast(hva, 1, 1, &page) == 1) { + write_ok = true; } else { - page = pages[0]; - pfn = page_to_pfn(page); - if (PageHuge(page)) { - page = compound_head(page); - pte_size <<= compound_order(page); - } - /* if the guest wants write access, see if that is OK */ - if (!writing && hpte_is_writable(r)) { - pte_t *ptep, pte; - unsigned long flags; - /* - * We need to protect against page table destruction - * hugepage split and collapse. - */ - local_irq_save(flags); - ptep = find_current_mm_pte(mm->pgd, hva, NULL, NULL); - if (ptep) { - pte = kvmppc_read_update_linux_pte(ptep, 1); - if (__pte_write(pte)) - write_ok = 1; - } - local_irq_restore(flags); + /* Call KVM generic code to do the slow-path check */ + pfn = __gfn_to_pfn_memslot(memslot, gfn, false, NULL, + writing, &write_ok); + if (is_error_noslot_pfn(pfn)) + return -EFAULT; + page = NULL; + if (pfn_valid(pfn)) { + page = pfn_to_page(pfn); + if (PageReserved(page)) + page = NULL; } } + /* + * Read the PTE from the process' radix tree and use that + * so we get the shift and attribute bits. + */ + local_irq_disable(); + ptep = __find_linux_pte(vcpu->arch.pgdir, hva, NULL, &shift); + /* + * If the PTE disappeared temporarily due to a THP + * collapse, just return and let the guest try again. + */ + if (!ptep) { + local_irq_enable(); + if (page) + put_page(page); + return RESUME_GUEST; + } + pte = *ptep; + local_irq_enable(); + hpa = pte_pfn(pte) << PAGE_SHIFT; + pte_size = PAGE_SIZE; + if (shift) + pte_size = 1ul << shift; + is_ci = pte_ci(pte); + if (psize > pte_size) goto out_put; + if (pte_size > psize) + hpa |= hva & (pte_size - psize); /* Check WIMG vs. the actual page we're accessing */ if (!hpte_cache_flags_ok(r, is_ci)) { @@ -636,14 +639,13 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, } /* - * Set the HPTE to point to pfn. - * Since the pfn is at PAGE_SIZE granularity, make sure we + * Set the HPTE to point to hpa. + * Since the hpa is at PAGE_SIZE granularity, make sure we * don't mask out lower-order bits if psize < PAGE_SIZE. */ if (psize < PAGE_SIZE) psize = PAGE_SIZE; - r = (r & HPTE_R_KEY_HI) | (r & ~(HPTE_R_PP0 - psize)) | - ((pfn << PAGE_SHIFT) & ~(psize - 1)); + r = (r & HPTE_R_KEY_HI) | (r & ~(HPTE_R_PP0 - psize)) | hpa; if (hpte_is_writable(r) && !write_ok) r = hpte_make_readonly(r); ret = RESUME_GUEST; @@ -708,20 +710,13 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, asm volatile("ptesync" : : : "memory"); preempt_enable(); if (page && hpte_is_writable(r)) - SetPageDirty(page); + set_page_dirty_lock(page); out_put: trace_kvm_page_fault_exit(vcpu, hpte, ret); - if (page) { - /* - * We drop pages[0] here, not page because page might - * have been set to the head page of a compound, but - * we have to drop the reference on the correct tail - * page to match the get inside gup() - */ - put_page(pages[0]); - } + if (page) + put_page(page); return ret; out_unlock: diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index 803940d79b73..134fbc1f029f 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -425,7 +425,7 @@ static void kvmppc_unmap_free_pte(struct kvm *kvm, pte_t *pte, bool full, unsigned int lpid) { if (full) { - memset(pte, 0, sizeof(long) << PTE_INDEX_SIZE); + memset(pte, 0, sizeof(long) << RADIX_PTE_INDEX_SIZE); } else { pte_t *p = pte; unsigned long it; diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index ee6c103bb7d5..50555ad1db93 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c index ab6eeb8e753e..6fcaf1fa8e02 100644 --- a/arch/powerpc/kvm/book3s_64_vio_hv.c +++ b/arch/powerpc/kvm/book3s_64_vio_hv.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 2cefd071b848..fa6e4fc7d0e4 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -72,7 +72,6 @@ #include #include #include -#include #include #include @@ -1074,25 +1073,35 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) kvmppc_get_gpr(vcpu, 6)); break; case H_SVM_PAGE_IN: - ret = kvmppc_h_svm_page_in(vcpu->kvm, - kvmppc_get_gpr(vcpu, 4), - kvmppc_get_gpr(vcpu, 5), - kvmppc_get_gpr(vcpu, 6)); + ret = H_UNSUPPORTED; + if (kvmppc_get_srr1(vcpu) & MSR_S) + ret = kvmppc_h_svm_page_in(vcpu->kvm, + kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5), + kvmppc_get_gpr(vcpu, 6)); break; case H_SVM_PAGE_OUT: - ret = kvmppc_h_svm_page_out(vcpu->kvm, - kvmppc_get_gpr(vcpu, 4), - kvmppc_get_gpr(vcpu, 5), - kvmppc_get_gpr(vcpu, 6)); + ret = H_UNSUPPORTED; + if (kvmppc_get_srr1(vcpu) & MSR_S) + ret = kvmppc_h_svm_page_out(vcpu->kvm, + kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5), + kvmppc_get_gpr(vcpu, 6)); break; case H_SVM_INIT_START: - ret = kvmppc_h_svm_init_start(vcpu->kvm); + ret = H_UNSUPPORTED; + if (kvmppc_get_srr1(vcpu) & MSR_S) + ret = kvmppc_h_svm_init_start(vcpu->kvm); break; case H_SVM_INIT_DONE: - ret = kvmppc_h_svm_init_done(vcpu->kvm); + ret = H_UNSUPPORTED; + if (kvmppc_get_srr1(vcpu) & MSR_S) + ret = kvmppc_h_svm_init_done(vcpu->kvm); break; case H_SVM_INIT_ABORT: - ret = kvmppc_h_svm_init_abort(vcpu->kvm); + ret = H_UNSUPPORTED; + if (kvmppc_get_srr1(vcpu) & MSR_S) + ret = kvmppc_h_svm_init_abort(vcpu->kvm); break; default: @@ -3616,6 +3625,7 @@ int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested && kvmppc_get_gpr(vcpu, 3) == H_CEDE) { kvmppc_nested_cede(vcpu); + kvmppc_set_gpr(vcpu, 3, 0); trap = 0; } } else { @@ -4400,7 +4410,7 @@ static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm, slots = kvm_memslots(kvm); memslot = id_to_memslot(slots, log->slot); r = -ENOENT; - if (!memslot->dirty_bitmap) + if (!memslot || !memslot->dirty_bitmap) goto out; /* @@ -4447,29 +4457,26 @@ static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm, return r; } -static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) +static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *slot) { - if (!dont || free->arch.rmap != dont->arch.rmap) { - vfree(free->arch.rmap); - free->arch.rmap = NULL; - } -} - -static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot, - unsigned long npages) -{ - slot->arch.rmap = vzalloc(array_size(npages, sizeof(*slot->arch.rmap))); - if (!slot->arch.rmap) - return -ENOMEM; - - return 0; + vfree(slot->arch.rmap); + slot->arch.rmap = NULL; } static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem) + struct kvm_memory_slot *slot, + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) { + unsigned long npages = mem->memory_size >> PAGE_SHIFT; + + if (change == KVM_MR_CREATE) { + slot->arch.rmap = vzalloc(array_size(npages, + sizeof(*slot->arch.rmap))); + if (!slot->arch.rmap) + return -ENOMEM; + } + return 0; } @@ -4558,11 +4565,6 @@ void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask) } } -static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu) -{ - return; -} - void kvmppc_setup_partition_table(struct kvm *kvm) { unsigned long dw0, dw1; @@ -5426,6 +5428,21 @@ static void unpin_vpa_reset(struct kvm *kvm, struct kvmppc_vpa *vpa) vpa->update_pending = 0; } +/* + * Enable a guest to become a secure VM, or test whether + * that could be enabled. + * Called when the KVM_CAP_PPC_SECURE_GUEST capability is + * tested (kvm == NULL) or enabled (kvm != NULL). + */ +static int kvmhv_enable_svm(struct kvm *kvm) +{ + if (!kvmppc_uvmem_available()) + return -EINVAL; + if (kvm) + kvm->arch.svm_enabled = 1; + return 0; +} + /* * IOCTL handler to turn off secure mode of guest * @@ -5526,9 +5543,7 @@ static struct kvmppc_ops kvm_ops_hv = { .age_hva = kvm_age_hva_hv, .test_age_hva = kvm_test_age_hva_hv, .set_spte_hva = kvm_set_spte_hva_hv, - .mmu_destroy = kvmppc_mmu_destroy_hv, .free_memslot = kvmppc_core_free_memslot_hv, - .create_memslot = kvmppc_core_create_memslot_hv, .init_vm = kvmppc_core_init_vm_hv, .destroy_vm = kvmppc_core_destroy_vm_hv, .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv, @@ -5548,6 +5563,7 @@ static struct kvmppc_ops kvm_ops_hv = { .enable_nested = kvmhv_enable_nested, .load_from_eaddr = kvmhv_load_from_eaddr, .store_to_eaddr = kvmhv_store_to_eaddr, + .enable_svm = kvmhv_enable_svm, .svm_off = kvmhv_svm_off, }; diff --git a/arch/powerpc/kvm/book3s_hv_tm.c b/arch/powerpc/kvm/book3s_hv_tm.c index 0db937497169..cc90b8b82329 100644 --- a/arch/powerpc/kvm/book3s_hv_tm.c +++ b/arch/powerpc/kvm/book3s_hv_tm.c @@ -3,6 +3,8 @@ * Copyright 2017 Paul Mackerras, IBM Corp. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include #include @@ -44,7 +46,18 @@ int kvmhv_p9_tm_emulation(struct kvm_vcpu *vcpu) u64 newmsr, bescr; int ra, rs; - switch (instr & 0xfc0007ff) { + /* + * rfid, rfebb, and mtmsrd encode bit 31 = 0 since it's a reserved bit + * in these instructions, so masking bit 31 out doesn't change these + * instructions. For treclaim., tsr., and trechkpt. instructions if bit + * 31 = 0 then they are per ISA invalid forms, however P9 UM, in section + * 4.6.10 Book II Invalid Forms, informs specifically that ignoring bit + * 31 is an acceptable way to handle these invalid forms that have + * bit 31 = 0. Moreover, for emulation purposes both forms (w/ and wo/ + * bit 31 set) can generate a softpatch interrupt. Hence both forms + * are handled below for these instructions so they behave the same way. + */ + switch (instr & PO_XOP_OPCODE_MASK) { case PPC_INST_RFID: /* XXX do we need to check for PR=0 here? */ newmsr = vcpu->arch.shregs.srr1; @@ -105,7 +118,8 @@ int kvmhv_p9_tm_emulation(struct kvm_vcpu *vcpu) vcpu->arch.shregs.msr = newmsr; return RESUME_GUEST; - case PPC_INST_TSR: + /* ignore bit 31, see comment above */ + case (PPC_INST_TSR & PO_XOP_OPCODE_MASK): /* check for PR=1 and arch 2.06 bit set in PCR */ if ((msr & MSR_PR) && (vcpu->arch.vcore->pcr & PCR_ARCH_206)) { /* generate an illegal instruction interrupt */ @@ -140,7 +154,8 @@ int kvmhv_p9_tm_emulation(struct kvm_vcpu *vcpu) vcpu->arch.shregs.msr = msr; return RESUME_GUEST; - case PPC_INST_TRECLAIM: + /* ignore bit 31, see comment above */ + case (PPC_INST_TRECLAIM & PO_XOP_OPCODE_MASK): /* check for TM disabled in the HFSCR or MSR */ if (!(vcpu->arch.hfscr & HFSCR_TM)) { /* generate an illegal instruction interrupt */ @@ -176,7 +191,8 @@ int kvmhv_p9_tm_emulation(struct kvm_vcpu *vcpu) vcpu->arch.shregs.msr &= ~MSR_TS_MASK; return RESUME_GUEST; - case PPC_INST_TRECHKPT: + /* ignore bit 31, see comment above */ + case (PPC_INST_TRECHKPT & PO_XOP_OPCODE_MASK): /* XXX do we need to check for PR=0 here? */ /* check for TM disabled in the HFSCR or MSR */ if (!(vcpu->arch.hfscr & HFSCR_TM)) { @@ -208,6 +224,8 @@ int kvmhv_p9_tm_emulation(struct kvm_vcpu *vcpu) } /* What should we do here? We didn't recognize the instruction */ - WARN_ON_ONCE(1); + kvmppc_core_queue_program(vcpu, SRR1_PROGILL); + pr_warn_ratelimited("Unrecognized TM-related instruction %#x for emulation", instr); + return RESUME_GUEST; } diff --git a/arch/powerpc/kvm/book3s_hv_tm_builtin.c b/arch/powerpc/kvm/book3s_hv_tm_builtin.c index 217246279dfa..fad931f224ef 100644 --- a/arch/powerpc/kvm/book3s_hv_tm_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_tm_builtin.c @@ -23,7 +23,18 @@ int kvmhv_p9_tm_emulation_early(struct kvm_vcpu *vcpu) u64 newmsr, msr, bescr; int rs; - switch (instr & 0xfc0007ff) { + /* + * rfid, rfebb, and mtmsrd encode bit 31 = 0 since it's a reserved bit + * in these instructions, so masking bit 31 out doesn't change these + * instructions. For the tsr. instruction if bit 31 = 0 then it is per + * ISA an invalid form, however P9 UM, in section 4.6.10 Book II Invalid + * Forms, informs specifically that ignoring bit 31 is an acceptable way + * to handle TM-related invalid forms that have bit 31 = 0. Moreover, + * for emulation purposes both forms (w/ and wo/ bit 31 set) can + * generate a softpatch interrupt. Hence both forms are handled below + * for tsr. to make them behave the same way. + */ + switch (instr & PO_XOP_OPCODE_MASK) { case PPC_INST_RFID: /* XXX do we need to check for PR=0 here? */ newmsr = vcpu->arch.shregs.srr1; @@ -73,7 +84,8 @@ int kvmhv_p9_tm_emulation_early(struct kvm_vcpu *vcpu) vcpu->arch.shregs.msr = newmsr; return 1; - case PPC_INST_TSR: + /* ignore bit 31, see comment above */ + case (PPC_INST_TSR & PO_XOP_OPCODE_MASK): /* we know the MSR has the TS field = S (0b01) here */ msr = vcpu->arch.shregs.msr; /* check for PR=1 and arch 2.06 bit set in PCR */ diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index f44f6b27950f..76d05c71fb1f 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -113,6 +113,15 @@ struct kvmppc_uvmem_page_pvt { bool skip_page_out; }; +bool kvmppc_uvmem_available(void) +{ + /* + * If kvmppc_uvmem_bitmap != NULL, then there is an ultravisor + * and our data structures have been initialized successfully. + */ + return !!kvmppc_uvmem_bitmap; +} + int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot) { struct kvmppc_uvmem_slot *p; @@ -209,6 +218,8 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) int ret = H_SUCCESS; int srcu_idx; + kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START; + if (!kvmppc_uvmem_bitmap) return H_UNSUPPORTED; @@ -216,6 +227,10 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) if (!kvm_is_radix(kvm)) return H_UNSUPPORTED; + /* NAK the transition to secure if not enabled */ + if (!kvm->arch.svm_enabled) + return H_AUTHORITY; + srcu_idx = srcu_read_lock(&kvm->srcu); slots = kvm_memslots(kvm); kvm_for_each_memslot(memslot, slots) { @@ -233,7 +248,6 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) goto out; } } - kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_START; out: srcu_read_unlock(&kvm->srcu, srcu_idx); return ret; @@ -809,6 +823,9 @@ int kvmppc_uvmem_init(void) void kvmppc_uvmem_free(void) { + if (!kvmppc_uvmem_bitmap) + return; + memunmap_pages(&kvmppc_uvmem_pgmap); release_mem_region(kvmppc_uvmem_pgmap.res.start, resource_size(&kvmppc_uvmem_pgmap.res)); diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c index db3a87319642..a0f6813f4560 100644 --- a/arch/powerpc/kvm/book3s_pr.c +++ b/arch/powerpc/kvm/book3s_pr.c @@ -740,7 +740,7 @@ int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu, (vcpu->arch.hflags & BOOK3S_HFLAG_SPLIT_HACK) && ((pte.raddr & SPLIT_HACK_MASK) == SPLIT_HACK_OFFS)) pte.raddr &= ~SPLIT_HACK_MASK; - /* fall through */ + fallthrough; case MSR_IR: vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid); @@ -1795,7 +1795,7 @@ static int kvmppc_core_vcpu_create_pr(struct kvm_vcpu *vcpu) vcpu->arch.shadow_msr = MSR_USER64 & ~MSR_LE; - err = kvmppc_mmu_init(vcpu); + err = kvmppc_mmu_init_pr(vcpu); if (err < 0) goto free_shared_page; @@ -1885,7 +1885,6 @@ static int kvmppc_vcpu_run_pr(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) static int kvm_vm_ioctl_get_dirty_log_pr(struct kvm *kvm, struct kvm_dirty_log *log) { - struct kvm_memslots *slots; struct kvm_memory_slot *memslot; struct kvm_vcpu *vcpu; ulong ga, ga_end; @@ -1895,15 +1894,12 @@ static int kvm_vm_ioctl_get_dirty_log_pr(struct kvm *kvm, mutex_lock(&kvm->slots_lock); - r = kvm_get_dirty_log(kvm, log, &is_dirty); + r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot); if (r) goto out; /* If nothing is dirty, don't bother messing with page tables. */ if (is_dirty) { - slots = kvm_memslots(kvm); - memslot = id_to_memslot(slots, log->slot); - ga = memslot->base_gfn << PAGE_SHIFT; ga_end = ga + (memslot->npages << PAGE_SHIFT); @@ -1928,7 +1924,8 @@ static void kvmppc_core_flush_memslot_pr(struct kvm *kvm, static int kvmppc_core_prepare_memory_region_pr(struct kvm *kvm, struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem) + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) { return 0; } @@ -1942,19 +1939,11 @@ static void kvmppc_core_commit_memory_region_pr(struct kvm *kvm, return; } -static void kvmppc_core_free_memslot_pr(struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) +static void kvmppc_core_free_memslot_pr(struct kvm_memory_slot *slot) { return; } -static int kvmppc_core_create_memslot_pr(struct kvm_memory_slot *slot, - unsigned long npages) -{ - return 0; -} - - #ifdef CONFIG_PPC64 static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm *kvm, struct kvm_ppc_smmu_info *info) @@ -2098,9 +2087,7 @@ static struct kvmppc_ops kvm_ops_pr = { .age_hva = kvm_age_hva_pr, .test_age_hva = kvm_test_age_hva_pr, .set_spte_hva = kvm_set_spte_hva_pr, - .mmu_destroy = kvmppc_mmu_destroy_pr, .free_memslot = kvmppc_core_free_memslot_pr, - .create_memslot = kvmppc_core_create_memslot_pr, .init_vm = kvmppc_core_init_vm_pr, .destroy_vm = kvmppc_core_destroy_vm_pr, .get_smmu_info = kvm_vm_ioctl_get_smmu_info_pr, diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index 7b27604adadf..6c18ea88fd25 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -421,11 +421,11 @@ static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu, case BOOKE_IRQPRIO_DATA_STORAGE: case BOOKE_IRQPRIO_ALIGNMENT: update_dear = true; - /* fall through */ + fallthrough; case BOOKE_IRQPRIO_INST_STORAGE: case BOOKE_IRQPRIO_PROGRAM: update_esr = true; - /* fall through */ + fallthrough; case BOOKE_IRQPRIO_ITLB_MISS: case BOOKE_IRQPRIO_SYSCALL: case BOOKE_IRQPRIO_FP_UNAVAIL: @@ -459,7 +459,7 @@ static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu, case BOOKE_IRQPRIO_DECREMENTER: case BOOKE_IRQPRIO_FIT: keep_irq = true; - /* fall through */ + fallthrough; case BOOKE_IRQPRIO_EXTERNAL: case BOOKE_IRQPRIO_DBELL: allowed = vcpu->arch.shared->msr & MSR_EE; @@ -1766,25 +1766,24 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, return r; } +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + +} + int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) { return -ENOTSUPP; } -void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) +void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { } -int kvmppc_core_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned long npages) -{ - return 0; -} - int kvmppc_core_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem) + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) { return 0; } @@ -2074,11 +2073,6 @@ void kvmppc_booke_vcpu_put(struct kvm_vcpu *vcpu) kvmppc_clear_dbsr(); } -void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu) -{ - vcpu->kvm->arch.kvm_ops->mmu_destroy(vcpu); -} - int kvmppc_core_init_vm(struct kvm *kvm) { return kvm->arch.kvm_ops->init_vm(kvm); diff --git a/arch/powerpc/kvm/booke.h b/arch/powerpc/kvm/booke.h index 9d3169fbce55..65b4d337d337 100644 --- a/arch/powerpc/kvm/booke.h +++ b/arch/powerpc/kvm/booke.h @@ -94,7 +94,6 @@ enum int_class { void kvmppc_set_pending_interrupt(struct kvm_vcpu *vcpu, enum int_class type); -extern void kvmppc_mmu_destroy_e500(struct kvm_vcpu *vcpu); extern int kvmppc_core_emulate_op_e500(struct kvm_run *run, struct kvm_vcpu *vcpu, unsigned int inst, int *advance); @@ -102,7 +101,6 @@ extern int kvmppc_core_emulate_mtspr_e500(struct kvm_vcpu *vcpu, int sprn, ulong spr_val); extern int kvmppc_core_emulate_mfspr_e500(struct kvm_vcpu *vcpu, int sprn, ulong *spr_val); -extern void kvmppc_mmu_destroy_e500(struct kvm_vcpu *vcpu); extern int kvmppc_core_emulate_op_e500(struct kvm_run *run, struct kvm_vcpu *vcpu, unsigned int inst, int *advance); diff --git a/arch/powerpc/kvm/e500.c b/arch/powerpc/kvm/e500.c index f2b4feaff6d2..7e8b69015d20 100644 --- a/arch/powerpc/kvm/e500.c +++ b/arch/powerpc/kvm/e500.c @@ -490,7 +490,6 @@ static struct kvmppc_ops kvm_ops_e500 = { .vcpu_put = kvmppc_core_vcpu_put_e500, .vcpu_create = kvmppc_core_vcpu_create_e500, .vcpu_free = kvmppc_core_vcpu_free_e500, - .mmu_destroy = kvmppc_mmu_destroy_e500, .init_vm = kvmppc_core_init_vm_e500, .destroy_vm = kvmppc_core_destroy_vm_e500, .emulate_op = kvmppc_core_emulate_op_e500, diff --git a/arch/powerpc/kvm/e500_mmu.c b/arch/powerpc/kvm/e500_mmu.c index 2d910b87e441..e131fbecdcc4 100644 --- a/arch/powerpc/kvm/e500_mmu.c +++ b/arch/powerpc/kvm/e500_mmu.c @@ -533,10 +533,6 @@ gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index, return get_tlb_raddr(gtlbe) | (eaddr & pgmask); } -void kvmppc_mmu_destroy_e500(struct kvm_vcpu *vcpu) -{ -} - /*****************************************/ static void free_gtlb(struct kvmppc_vcpu_e500 *vcpu_e500) diff --git a/arch/powerpc/kvm/e500mc.c b/arch/powerpc/kvm/e500mc.c index e6b06cb2b92c..1c189b5aadcc 100644 --- a/arch/powerpc/kvm/e500mc.c +++ b/arch/powerpc/kvm/e500mc.c @@ -376,7 +376,6 @@ static struct kvmppc_ops kvm_ops_e500mc = { .vcpu_put = kvmppc_core_vcpu_put_e500mc, .vcpu_create = kvmppc_core_vcpu_create_e500mc, .vcpu_free = kvmppc_core_vcpu_free_e500mc, - .mmu_destroy = kvmppc_mmu_destroy_e500, .init_vm = kvmppc_core_init_vm_e500mc, .destroy_vm = kvmppc_core_destroy_vm_e500mc, .emulate_op = kvmppc_core_emulate_op_e500, diff --git a/arch/powerpc/kvm/mpic.c b/arch/powerpc/kvm/mpic.c index fe312c160d97..23e9c2bd9f27 100644 --- a/arch/powerpc/kvm/mpic.c +++ b/arch/powerpc/kvm/mpic.c @@ -32,7 +32,6 @@ #include #include #include -#include #include #include diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 302e9dccdd6d..e15166b0a16d 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -32,7 +32,6 @@ #include #endif #include -#include #include "timing.h" #include "irq.h" @@ -416,12 +415,12 @@ int kvm_arch_hardware_enable(void) return 0; } -int kvm_arch_hardware_setup(void) +int kvm_arch_hardware_setup(void *opaque) { return 0; } -int kvm_arch_check_processor_compat(void) +int kvm_arch_check_processor_compat(void *opaque) { return kvmppc_core_check_processor_compat(); } @@ -525,7 +524,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = 1; break; case KVM_CAP_PPC_GUEST_DEBUG_SSTEP: - /* fall through */ case KVM_CAP_PPC_PAIRED_SINGLES: case KVM_CAP_PPC_OSI: case KVM_CAP_PPC_GET_PVINFO: @@ -670,6 +668,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = !!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_HTM) || (hv_enabled && cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)); break; +#endif +#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) + case KVM_CAP_PPC_SECURE_GUEST: + r = hv_enabled && kvmppc_hv_ops->enable_svm && + !kvmppc_hv_ops->enable_svm(NULL); + break; #endif default: r = 0; @@ -685,16 +689,9 @@ long kvm_arch_dev_ioctl(struct file *filp, return -EINVAL; } -void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) +void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { - kvmppc_core_free_memslot(kvm, free, dont); -} - -int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned long npages) -{ - return kvmppc_core_create_memslot(kvm, slot, npages); + kvmppc_core_free_memslot(kvm, slot); } int kvm_arch_prepare_memory_region(struct kvm *kvm, @@ -702,12 +699,12 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, enum kvm_mr_change change) { - return kvmppc_core_prepare_memory_region(kvm, memslot, mem); + return kvmppc_core_prepare_memory_region(kvm, memslot, mem, change); } void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { @@ -2175,6 +2172,14 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, break; r = kvm->arch.kvm_ops->enable_nested(kvm); break; +#endif +#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) + case KVM_CAP_PPC_SECURE_GUEST: + r = -EINVAL; + if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_svm) + break; + r = kvm->arch.kvm_ops->enable_svm(kvm); + break; #endif default: r = -EINVAL; diff --git a/arch/powerpc/kvm/timing.h b/arch/powerpc/kvm/timing.h index ace65f9fed30..feef7885ba82 100644 --- a/arch/powerpc/kvm/timing.h +++ b/arch/powerpc/kvm/timing.h @@ -10,7 +10,6 @@ #define __POWERPC_KVM_EXITTIMING_H__ #include -#include #ifdef CONFIG_KVM_EXIT_TIMING void kvmppc_init_timing_stats(struct kvm_vcpu *vcpu); diff --git a/arch/powerpc/mm/book3s64/pkeys.c b/arch/powerpc/mm/book3s64/pkeys.c index 59e0ebbd8036..07527f1ed108 100644 --- a/arch/powerpc/mm/book3s64/pkeys.c +++ b/arch/powerpc/mm/book3s64/pkeys.c @@ -381,18 +381,6 @@ bool arch_pte_access_permitted(u64 pte, bool write, bool execute) * So do not enforce things if the VMA is not from the current mm, or if we are * in a kernel thread. */ -static inline bool vma_is_foreign(struct vm_area_struct *vma) -{ - if (!current->mm) - return true; - - /* if it is not our ->mm, it has to be foreign */ - if (current->mm != vma->vm_mm) - return true; - - return false; -} - bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, bool execute, bool foreign) { diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 8db0507619e2..d15f0f0ee806 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -434,7 +434,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, { struct vm_area_struct * vma; struct mm_struct *mm = current->mm; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; int is_exec = TRAP(regs) == 0x400; int is_user = user_mode(regs); int is_write = page_fault_is_write(error_code); @@ -582,28 +582,18 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, major |= fault & VM_FAULT_MAJOR; + if (fault_signal_pending(fault, regs)) + return user_mode(regs) ? 0 : SIGBUS; + /* * Handle the retry right now, the mmap_sem has been released in that * case. */ if (unlikely(fault & VM_FAULT_RETRY)) { - /* We retry only once */ if (flags & FAULT_FLAG_ALLOW_RETRY) { - /* - * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk - * of starvation. - */ - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; - if (!fatal_signal_pending(current)) - goto retry; + goto retry; } - - /* - * User mode? Just return to handle the fatal exception otherwise - * return to bad_page_fault - */ - return is_user ? 0 : SIGBUS; } up_read(¤t->mm->mmap_sem); diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c index a4d40a3ceea3..f3ed1baa6289 100644 --- a/arch/powerpc/platforms/pseries/hotplug-memory.c +++ b/arch/powerpc/platforms/pseries/hotplug-memory.c @@ -360,7 +360,7 @@ static bool lmb_is_removable(struct drmem_lmb *lmb) for (i = 0; i < scns_per_block; i++) { pfn = PFN_DOWN(phys_addr); - if (!pfn_present(pfn)) { + if (!pfn_in_present_section(pfn)) { phys_addr += MIN_MEMORY_BLOCK_SIZE; continue; } diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index ec0ca8c6ab64..3d9410bb4de0 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -1,35 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -generic-y += bugs.h -generic-y += checksum.h -generic-y += compat.h -generic-y += device.h -generic-y += div64.h generic-y += extable.h generic-y += flat.h -generic-y += dma.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h -generic-y += fb.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h generic-y += kvm_para.h -generic-y += local.h generic-y += local64.h -generic-y += mm-arch-hooks.h -generic-y += percpu.h -generic-y += preempt.h -generic-y += sections.h -generic-y += serial.h -generic-y += shmparam.h -generic-y += topology.h -generic-y += trace_clock.h -generic-y += unaligned.h generic-y += user.h -generic-y += vga.h generic-y += vmlinux.lds.h -generic-y += xor.h diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index cf7248e07f43..be84e32adc4c 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -30,7 +30,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) struct vm_area_struct *vma; struct mm_struct *mm; unsigned long addr, cause; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; int code = SEGV_MAPERR; vm_fault_t fault; @@ -117,7 +117,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) * signal first. We do not need to release the mmap_sem because it * would already be released in __lock_page_or_retry in mm/filemap.c. */ - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(tsk)) + if (fault_signal_pending(fault, regs)) return; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -144,11 +144,6 @@ asmlinkage void do_page_fault(struct pt_regs *regs) 1, regs, addr); } if (fault & VM_FAULT_RETRY) { - /* - * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk - * of starvation. - */ - flags &= ~(FAULT_FLAG_ALLOW_RETRY); flags |= FAULT_FLAG_TRIED; /* diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 8abe77536d9d..59c216af6264 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -195,6 +195,7 @@ config S390 select ARCH_HAS_FORCE_DMA_UNENCRYPTED select SWIOTLB select GENERIC_ALLOCATOR + imply IMA_SECURE_AND_OR_TRUSTED_BOOT config SCHED_OMIT_FRAME_POINTER diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index 0ff9261c915e..45b33b83de08 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -37,7 +37,7 @@ CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char obj-y := head.o als.o startup.o mem_detect.o ipl_parm.o ipl_report.o obj-y += string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o obj-y += version.o pgm_check_info.o ctype.o text_dma.o -obj-$(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) += uv.o +obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_PGSTE)) += uv.o obj-$(CONFIG_RELOCATABLE) += machine_kexec_reloc.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o targets := bzImage startup.a section_cmp.boot.data section_cmp.boot.preserved.data $(obj-y) diff --git a/arch/s390/boot/uv.c b/arch/s390/boot/uv.c index 3f501159ee9f..8fde561f1d07 100644 --- a/arch/s390/boot/uv.c +++ b/arch/s390/boot/uv.c @@ -3,7 +3,13 @@ #include #include +/* will be used in arch/s390/kernel/uv.c */ +#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST int __bootdata_preserved(prot_virt_guest); +#endif +#if IS_ENABLED(CONFIG_KVM) +struct uv_info __bootdata_preserved(uv_info); +#endif void uv_query_info(void) { @@ -19,7 +25,21 @@ void uv_query_info(void) if (uv_call(0, (uint64_t)&uvcb) && uvcb.header.rc != 0x100) return; + if (IS_ENABLED(CONFIG_KVM)) { + memcpy(uv_info.inst_calls_list, uvcb.inst_calls_list, sizeof(uv_info.inst_calls_list)); + uv_info.uv_base_stor_len = uvcb.uv_base_stor_len; + uv_info.guest_base_stor_len = uvcb.conf_base_phys_stor_len; + uv_info.guest_virt_base_stor_len = uvcb.conf_base_virt_stor_len; + uv_info.guest_virt_var_stor_len = uvcb.conf_virt_var_stor_len; + uv_info.guest_cpu_stor_len = uvcb.cpu_stor_len; + uv_info.max_sec_stor_addr = ALIGN(uvcb.max_guest_stor_addr, PAGE_SIZE); + uv_info.max_num_sec_conf = uvcb.max_num_sec_conf; + uv_info.max_guest_cpus = uvcb.max_guest_cpus; + } + +#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST if (test_bit_inv(BIT_UVC_CMD_SET_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list) && test_bit_inv(BIT_UVC_CMD_REMOVE_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list)) prot_virt_guest = 1; +#endif } diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild index 1832ae6442ef..83f6e85de7bc 100644 --- a/arch/s390/include/asm/Kbuild +++ b/arch/s390/include/asm/Kbuild @@ -5,21 +5,6 @@ generated-y += syscall_table.h generated-y += unistd_nr.h generic-y += asm-offsets.h -generic-y += cacheflush.h -generic-y += device.h -generic-y += dma-mapping.h -generic-y += div64.h -generic-y += emergency-restart.h generic-y += export.h -generic-y += fb.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kmap_types.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += trace_clock.h -generic-y += unaligned.h -generic-y += word-at-a-time.h diff --git a/arch/s390/include/asm/gmap.h b/arch/s390/include/asm/gmap.h index 37f96b6f0e61..a816fb4734b8 100644 --- a/arch/s390/include/asm/gmap.h +++ b/arch/s390/include/asm/gmap.h @@ -9,6 +9,7 @@ #ifndef _ASM_S390_GMAP_H #define _ASM_S390_GMAP_H +#include #include /* Generic bits for GMAP notification on DAT table entry changes. */ @@ -31,6 +32,7 @@ * @table: pointer to the page directory * @asce: address space control element for gmap page table * @pfault_enabled: defines if pfaults are applicable for the guest + * @guest_handle: protected virtual machine handle for the ultravisor * @host_to_rmap: radix tree with gmap_rmap lists * @children: list of shadow gmap structures * @pt_list: list of all page tables used in the shadow guest address space @@ -54,6 +56,8 @@ struct gmap { unsigned long asce_end; void *private; bool pfault_enabled; + /* only set for protected virtual machines */ + unsigned long guest_handle; /* Additional data for shadow guest address spaces */ struct radix_tree_root host_to_rmap; struct list_head children; @@ -144,4 +148,6 @@ int gmap_mprotect_notify(struct gmap *, unsigned long start, void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long dirty_bitmap[4], unsigned long gaddr, unsigned long vmaddr); +int gmap_mark_unmergeable(void); +void s390_reset_acc(struct mm_struct *mm); #endif /* _ASM_S390_GMAP_H */ diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 1726224e7772..d6bcd34f3ec3 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -127,6 +127,12 @@ struct mcck_volatile_info { #define CR14_INITIAL_MASK (CR14_UNUSED_32 | CR14_UNUSED_33 | \ CR14_EXTERNAL_DAMAGE_SUBMASK) +#define SIDAD_SIZE_MASK 0xff +#define sida_origin(sie_block) \ + ((sie_block)->sidad & PAGE_MASK) +#define sida_size(sie_block) \ + ((((sie_block)->sidad & SIDAD_SIZE_MASK) + 1) * PAGE_SIZE) + #define CPUSTAT_STOPPED 0x80000000 #define CPUSTAT_WAIT 0x10000000 #define CPUSTAT_ECALL_PEND 0x08000000 @@ -160,7 +166,13 @@ struct kvm_s390_sie_block { __u8 reserved08[4]; /* 0x0008 */ #define PROG_IN_SIE (1<<0) __u32 prog0c; /* 0x000c */ - __u8 reserved10[16]; /* 0x0010 */ + union { + __u8 reserved10[16]; /* 0x0010 */ + struct { + __u64 pv_handle_cpu; + __u64 pv_handle_config; + }; + }; #define PROG_BLOCK_SIE (1<<0) #define PROG_REQUEST (1<<1) atomic_t prog20; /* 0x0020 */ @@ -209,10 +221,23 @@ struct kvm_s390_sie_block { #define ICPT_PARTEXEC 0x38 #define ICPT_IOINST 0x40 #define ICPT_KSS 0x5c +#define ICPT_MCHKREQ 0x60 +#define ICPT_INT_ENABLE 0x64 +#define ICPT_PV_INSTR 0x68 +#define ICPT_PV_NOTIFY 0x6c +#define ICPT_PV_PREF 0x70 __u8 icptcode; /* 0x0050 */ __u8 icptstatus; /* 0x0051 */ __u16 ihcpu; /* 0x0052 */ - __u8 reserved54[2]; /* 0x0054 */ + __u8 reserved54; /* 0x0054 */ +#define IICTL_CODE_NONE 0x00 +#define IICTL_CODE_MCHK 0x01 +#define IICTL_CODE_EXT 0x02 +#define IICTL_CODE_IO 0x03 +#define IICTL_CODE_RESTART 0x04 +#define IICTL_CODE_SPECIFICATION 0x10 +#define IICTL_CODE_OPERAND 0x11 + __u8 iictl; /* 0x0055 */ __u16 ipa; /* 0x0056 */ __u32 ipb; /* 0x0058 */ __u32 scaoh; /* 0x005c */ @@ -233,7 +258,7 @@ struct kvm_s390_sie_block { #define ECB3_RI 0x01 __u8 ecb3; /* 0x0063 */ __u32 scaol; /* 0x0064 */ - __u8 reserved68; /* 0x0068 */ + __u8 sdf; /* 0x0068 */ __u8 epdx; /* 0x0069 */ __u8 reserved6a[2]; /* 0x006a */ __u32 todpr; /* 0x006c */ @@ -249,31 +274,58 @@ struct kvm_s390_sie_block { #define HPID_KVM 0x4 #define HPID_VSIE 0x5 __u8 hpid; /* 0x00b8 */ - __u8 reservedb9[11]; /* 0x00b9 */ - __u16 extcpuaddr; /* 0x00c4 */ - __u16 eic; /* 0x00c6 */ + __u8 reservedb9[7]; /* 0x00b9 */ + union { + struct { + __u32 eiparams; /* 0x00c0 */ + __u16 extcpuaddr; /* 0x00c4 */ + __u16 eic; /* 0x00c6 */ + }; + __u64 mcic; /* 0x00c0 */ + } __packed; __u32 reservedc8; /* 0x00c8 */ - __u16 pgmilc; /* 0x00cc */ - __u16 iprcc; /* 0x00ce */ - __u32 dxc; /* 0x00d0 */ - __u16 mcn; /* 0x00d4 */ - __u8 perc; /* 0x00d6 */ - __u8 peratmid; /* 0x00d7 */ + union { + struct { + __u16 pgmilc; /* 0x00cc */ + __u16 iprcc; /* 0x00ce */ + }; + __u32 edc; /* 0x00cc */ + } __packed; + union { + struct { + __u32 dxc; /* 0x00d0 */ + __u16 mcn; /* 0x00d4 */ + __u8 perc; /* 0x00d6 */ + __u8 peratmid; /* 0x00d7 */ + }; + __u64 faddr; /* 0x00d0 */ + } __packed; __u64 peraddr; /* 0x00d8 */ __u8 eai; /* 0x00e0 */ __u8 peraid; /* 0x00e1 */ __u8 oai; /* 0x00e2 */ __u8 armid; /* 0x00e3 */ __u8 reservede4[4]; /* 0x00e4 */ - __u64 tecmc; /* 0x00e8 */ - __u8 reservedf0[12]; /* 0x00f0 */ + union { + __u64 tecmc; /* 0x00e8 */ + struct { + __u16 subchannel_id; /* 0x00e8 */ + __u16 subchannel_nr; /* 0x00ea */ + __u32 io_int_parm; /* 0x00ec */ + __u32 io_int_word; /* 0x00f0 */ + }; + } __packed; + __u8 reservedf4[8]; /* 0x00f4 */ #define CRYCB_FORMAT_MASK 0x00000003 #define CRYCB_FORMAT0 0x00000000 #define CRYCB_FORMAT1 0x00000001 #define CRYCB_FORMAT2 0x00000003 __u32 crycbd; /* 0x00fc */ __u64 gcr[16]; /* 0x0100 */ - __u64 gbea; /* 0x0180 */ + union { + __u64 gbea; /* 0x0180 */ + __u64 sidad; + }; __u8 reserved188[8]; /* 0x0188 */ __u64 sdnxo; /* 0x0190 */ __u8 reserved198[8]; /* 0x0198 */ @@ -292,7 +344,7 @@ struct kvm_s390_sie_block { __u64 itdba; /* 0x01e8 */ __u64 riccbd; /* 0x01f0 */ __u64 gvrd; /* 0x01f8 */ -} __attribute__((packed)); +} __packed __aligned(512); struct kvm_s390_itdb { __u8 data[256]; @@ -301,7 +353,9 @@ struct kvm_s390_itdb { struct sie_page { struct kvm_s390_sie_block sie_block; struct mcck_volatile_info mcck_info; /* 0x0200 */ - __u8 reserved218[1000]; /* 0x0218 */ + __u8 reserved218[360]; /* 0x0218 */ + __u64 pv_grregs[16]; /* 0x0380 */ + __u8 reserved400[512]; /* 0x0400 */ struct kvm_s390_itdb itdb; /* 0x0600 */ __u8 reserved700[2304]; /* 0x0700 */ }; @@ -476,6 +530,7 @@ enum irq_types { IRQ_PEND_PFAULT_INIT, IRQ_PEND_EXT_HOST, IRQ_PEND_EXT_SERVICE, + IRQ_PEND_EXT_SERVICE_EV, IRQ_PEND_EXT_TIMING, IRQ_PEND_EXT_CPU_TIMER, IRQ_PEND_EXT_CLOCK_COMP, @@ -520,6 +575,7 @@ enum irq_types { (1UL << IRQ_PEND_EXT_TIMING) | \ (1UL << IRQ_PEND_EXT_HOST) | \ (1UL << IRQ_PEND_EXT_SERVICE) | \ + (1UL << IRQ_PEND_EXT_SERVICE_EV) | \ (1UL << IRQ_PEND_VIRTIO) | \ (1UL << IRQ_PEND_PFAULT_INIT) | \ (1UL << IRQ_PEND_PFAULT_DONE)) @@ -536,6 +592,13 @@ enum irq_types { #define IRQ_PEND_MCHK_MASK ((1UL << IRQ_PEND_MCHK_REP) | \ (1UL << IRQ_PEND_MCHK_EX)) +#define IRQ_PEND_EXT_II_MASK ((1UL << IRQ_PEND_EXT_CPU_TIMER) | \ + (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ + (1UL << IRQ_PEND_EXT_EMERGENCY) | \ + (1UL << IRQ_PEND_EXT_EXTERNAL) | \ + (1UL << IRQ_PEND_EXT_SERVICE) | \ + (1UL << IRQ_PEND_EXT_SERVICE_EV)) + struct kvm_s390_interrupt_info { struct list_head list; u64 type; @@ -594,6 +657,7 @@ struct kvm_s390_local_interrupt { struct kvm_s390_float_interrupt { unsigned long pending_irqs; + unsigned long masked_irqs; spinlock_t lock; struct list_head lists[FIRQ_LIST_COUNT]; int counters[FIRQ_MAX_COUNT]; @@ -645,6 +709,11 @@ struct kvm_guestdbg_info_arch { unsigned long last_bp; }; +struct kvm_s390_pv_vcpu { + u64 handle; + unsigned long stor_base; +}; + struct kvm_vcpu_arch { struct kvm_s390_sie_block *sie_block; /* if vsie is active, currently executed shadow sie control block */ @@ -673,6 +742,7 @@ struct kvm_vcpu_arch { __u64 cputm_start; bool gs_enabled; bool skey_enabled; + struct kvm_s390_pv_vcpu pv; }; struct kvm_vm_stat { @@ -701,9 +771,6 @@ struct s390_io_adapter { bool masked; bool swap; bool suppressible; - struct rw_semaphore maps_lock; - struct list_head maps; - atomic_t nr_maps; }; #define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) @@ -846,6 +913,13 @@ struct kvm_s390_gisa_interrupt { DECLARE_BITMAP(kicked_mask, KVM_MAX_VCPUS); }; +struct kvm_s390_pv { + u64 handle; + u64 guest_len; + unsigned long stor_base; + void *stor_var; +}; + struct kvm_arch{ void *sca; int use_esca; @@ -881,6 +955,7 @@ struct kvm_arch{ DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS); struct kvm_s390_gisa_interrupt gisa_int; + struct kvm_s390_pv pv; }; #define KVM_HVA_ERR_BAD (-1UL) @@ -921,7 +996,7 @@ static inline void kvm_arch_hardware_disable(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, - struct kvm_memory_slot *free, struct kvm_memory_slot *dont) {} + struct kvm_memory_slot *slot) {} static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index bcfb6371086f..e21b618ad432 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -16,6 +16,8 @@ typedef struct { unsigned long asce; unsigned long asce_limit; unsigned long vdso_base; + /* The mmu context belongs to a secure guest. */ + atomic_t is_protected; /* * The following bitfields need a down_write on the mm * semaphore when they are written to. As they are only diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 8d04e6f3f796..afa836014076 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -23,6 +23,7 @@ static inline int init_new_context(struct task_struct *tsk, INIT_LIST_HEAD(&mm->context.gmap_list); cpumask_clear(&mm->context.cpu_attach_mask); atomic_set(&mm->context.flush_count, 0); + atomic_set(&mm->context.is_protected, 0); mm->context.gmap_asce = 0; mm->context.flush_mm = 0; mm->context.compat_mm = test_thread_flag(TIF_31BIT); diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h index 1019efd85b9d..62440a82731a 100644 --- a/arch/s390/include/asm/page.h +++ b/arch/s390/include/asm/page.h @@ -153,6 +153,11 @@ static inline int devmem_is_allowed(unsigned long pfn) #define HAVE_ARCH_FREE_PAGE #define HAVE_ARCH_ALLOC_PAGE +#if IS_ENABLED(CONFIG_PGSTE) +int arch_make_page_accessible(struct page *page); +#define HAVE_ARCH_MAKE_PAGE_ACCESSIBLE +#endif + #endif /* !__ASSEMBLY__ */ #define __PAGE_OFFSET 0x0UL diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 6d7c3b7e9281..6076c8c912d2 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -19,6 +19,7 @@ #include #include #include +#include extern pgd_t swapper_pg_dir[]; extern void paging_init(void); @@ -520,6 +521,15 @@ static inline int mm_has_pgste(struct mm_struct *mm) return 0; } +static inline int mm_is_protected(struct mm_struct *mm) +{ +#ifdef CONFIG_PGSTE + if (unlikely(atomic_read(&mm->context.is_protected))) + return 1; +#endif + return 0; +} + static inline int mm_alloc_pgste(struct mm_struct *mm) { #ifdef CONFIG_PGSTE @@ -1067,7 +1077,12 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); + pte_t res; + + res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); + if (mm_is_protected(mm) && pte_present(res)) + uv_convert_from_secure(pte_val(res) & PAGE_MASK); + return res; } #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION @@ -1079,7 +1094,12 @@ void ptep_modify_prot_commit(struct vm_area_struct *, unsigned long, static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { - return ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID)); + pte_t res; + + res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID)); + if (mm_is_protected(vma->vm_mm) && pte_present(res)) + uv_convert_from_secure(pte_val(res) & PAGE_MASK); + return res; } /* @@ -1094,12 +1114,17 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) { + pte_t res; + if (full) { - pte_t pte = *ptep; + res = *ptep; *ptep = __pte(_PAGE_INVALID); - return pte; + } else { + res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); } - return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); + if (mm_is_protected(mm) && pte_present(res)) + uv_convert_from_secure(pte_val(res) & PAGE_MASK); + return res; } #define __HAVE_ARCH_PTEP_SET_WRPROTECT diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h index 4093a2856929..cff4b4c99b75 100644 --- a/arch/s390/include/asm/uv.h +++ b/arch/s390/include/asm/uv.h @@ -14,23 +14,62 @@ #include #include #include +#include #include +#include #define UVC_RC_EXECUTED 0x0001 #define UVC_RC_INV_CMD 0x0002 #define UVC_RC_INV_STATE 0x0003 #define UVC_RC_INV_LEN 0x0005 #define UVC_RC_NO_RESUME 0x0007 +#define UVC_RC_NEED_DESTROY 0x8000 #define UVC_CMD_QUI 0x0001 +#define UVC_CMD_INIT_UV 0x000f +#define UVC_CMD_CREATE_SEC_CONF 0x0100 +#define UVC_CMD_DESTROY_SEC_CONF 0x0101 +#define UVC_CMD_CREATE_SEC_CPU 0x0120 +#define UVC_CMD_DESTROY_SEC_CPU 0x0121 +#define UVC_CMD_CONV_TO_SEC_STOR 0x0200 +#define UVC_CMD_CONV_FROM_SEC_STOR 0x0201 +#define UVC_CMD_SET_SEC_CONF_PARAMS 0x0300 +#define UVC_CMD_UNPACK_IMG 0x0301 +#define UVC_CMD_VERIFY_IMG 0x0302 +#define UVC_CMD_CPU_RESET 0x0310 +#define UVC_CMD_CPU_RESET_INITIAL 0x0311 +#define UVC_CMD_PREPARE_RESET 0x0320 +#define UVC_CMD_CPU_RESET_CLEAR 0x0321 +#define UVC_CMD_CPU_SET_STATE 0x0330 +#define UVC_CMD_SET_UNSHARE_ALL 0x0340 +#define UVC_CMD_PIN_PAGE_SHARED 0x0341 +#define UVC_CMD_UNPIN_PAGE_SHARED 0x0342 #define UVC_CMD_SET_SHARED_ACCESS 0x1000 #define UVC_CMD_REMOVE_SHARED_ACCESS 0x1001 /* Bits in installed uv calls */ enum uv_cmds_inst { BIT_UVC_CMD_QUI = 0, + BIT_UVC_CMD_INIT_UV = 1, + BIT_UVC_CMD_CREATE_SEC_CONF = 2, + BIT_UVC_CMD_DESTROY_SEC_CONF = 3, + BIT_UVC_CMD_CREATE_SEC_CPU = 4, + BIT_UVC_CMD_DESTROY_SEC_CPU = 5, + BIT_UVC_CMD_CONV_TO_SEC_STOR = 6, + BIT_UVC_CMD_CONV_FROM_SEC_STOR = 7, BIT_UVC_CMD_SET_SHARED_ACCESS = 8, BIT_UVC_CMD_REMOVE_SHARED_ACCESS = 9, + BIT_UVC_CMD_SET_SEC_PARMS = 11, + BIT_UVC_CMD_UNPACK_IMG = 13, + BIT_UVC_CMD_VERIFY_IMG = 14, + BIT_UVC_CMD_CPU_RESET = 15, + BIT_UVC_CMD_CPU_RESET_INITIAL = 16, + BIT_UVC_CMD_CPU_SET_STATE = 17, + BIT_UVC_CMD_PREPARE_RESET = 18, + BIT_UVC_CMD_CPU_PERFORM_CLEAR_RESET = 19, + BIT_UVC_CMD_UNSHARE_ALL = 20, + BIT_UVC_CMD_PIN_PAGE_SHARED = 21, + BIT_UVC_CMD_UNPIN_PAGE_SHARED = 22, }; struct uv_cb_header { @@ -40,13 +79,127 @@ struct uv_cb_header { u16 rrc; /* Return Reason Code */ } __packed __aligned(8); +/* Query Ultravisor Information */ struct uv_cb_qui { struct uv_cb_header header; u64 reserved08; u64 inst_calls_list[4]; - u64 reserved30[15]; + u64 reserved30[2]; + u64 uv_base_stor_len; + u64 reserved48; + u64 conf_base_phys_stor_len; + u64 conf_base_virt_stor_len; + u64 conf_virt_var_stor_len; + u64 cpu_stor_len; + u32 reserved70[3]; + u32 max_num_sec_conf; + u64 max_guest_stor_addr; + u8 reserved88[158 - 136]; + u16 max_guest_cpus; + u8 reserveda0[200 - 160]; } __packed __aligned(8); +/* Initialize Ultravisor */ +struct uv_cb_init { + struct uv_cb_header header; + u64 reserved08[2]; + u64 stor_origin; + u64 stor_len; + u64 reserved28[4]; +} __packed __aligned(8); + +/* Create Guest Configuration */ +struct uv_cb_cgc { + struct uv_cb_header header; + u64 reserved08[2]; + u64 guest_handle; + u64 conf_base_stor_origin; + u64 conf_virt_stor_origin; + u64 reserved30; + u64 guest_stor_origin; + u64 guest_stor_len; + u64 guest_sca; + u64 guest_asce; + u64 reserved58[5]; +} __packed __aligned(8); + +/* Create Secure CPU */ +struct uv_cb_csc { + struct uv_cb_header header; + u64 reserved08[2]; + u64 cpu_handle; + u64 guest_handle; + u64 stor_origin; + u8 reserved30[6]; + u16 num; + u64 state_origin; + u64 reserved40[4]; +} __packed __aligned(8); + +/* Convert to Secure */ +struct uv_cb_cts { + struct uv_cb_header header; + u64 reserved08[2]; + u64 guest_handle; + u64 gaddr; +} __packed __aligned(8); + +/* Convert from Secure / Pin Page Shared */ +struct uv_cb_cfs { + struct uv_cb_header header; + u64 reserved08[2]; + u64 paddr; +} __packed __aligned(8); + +/* Set Secure Config Parameter */ +struct uv_cb_ssc { + struct uv_cb_header header; + u64 reserved08[2]; + u64 guest_handle; + u64 sec_header_origin; + u32 sec_header_len; + u32 reserved2c; + u64 reserved30[4]; +} __packed __aligned(8); + +/* Unpack */ +struct uv_cb_unp { + struct uv_cb_header header; + u64 reserved08[2]; + u64 guest_handle; + u64 gaddr; + u64 tweak[2]; + u64 reserved38[3]; +} __packed __aligned(8); + +#define PV_CPU_STATE_OPR 1 +#define PV_CPU_STATE_STP 2 +#define PV_CPU_STATE_CHKSTP 3 +#define PV_CPU_STATE_OPR_LOAD 5 + +struct uv_cb_cpu_set_state { + struct uv_cb_header header; + u64 reserved08[2]; + u64 cpu_handle; + u8 reserved20[7]; + u8 state; + u64 reserved28[5]; +}; + +/* + * A common UV call struct for calls that take no payload + * Examples: + * Destroy cpu/config + * Verify + */ +struct uv_cb_nodata { + struct uv_cb_header header; + u64 reserved08[2]; + u64 handle; + u64 reserved20[4]; +} __packed __aligned(8); + +/* Set Shared Access */ struct uv_cb_share { struct uv_cb_header header; u64 reserved08[3]; @@ -54,21 +207,76 @@ struct uv_cb_share { u64 reserved28; } __packed __aligned(8); -static inline int uv_call(unsigned long r1, unsigned long r2) +static inline int __uv_call(unsigned long r1, unsigned long r2) { int cc; asm volatile( - "0: .insn rrf,0xB9A40000,%[r1],%[r2],0,0\n" - " brc 3,0b\n" - " ipm %[cc]\n" - " srl %[cc],28\n" + " .insn rrf,0xB9A40000,%[r1],%[r2],0,0\n" + " ipm %[cc]\n" + " srl %[cc],28\n" : [cc] "=d" (cc) : [r1] "a" (r1), [r2] "a" (r2) : "memory", "cc"); return cc; } +static inline int uv_call(unsigned long r1, unsigned long r2) +{ + int cc; + + do { + cc = __uv_call(r1, r2); + } while (cc > 1); + return cc; +} + +/* Low level uv_call that avoids stalls for long running busy conditions */ +static inline int uv_call_sched(unsigned long r1, unsigned long r2) +{ + int cc; + + do { + cc = __uv_call(r1, r2); + cond_resched(); + } while (cc > 1); + return cc; +} + +/* + * special variant of uv_call that only transports the cpu or guest + * handle and the command, like destroy or verify. + */ +static inline int uv_cmd_nodata(u64 handle, u16 cmd, u16 *rc, u16 *rrc) +{ + struct uv_cb_nodata uvcb = { + .header.cmd = cmd, + .header.len = sizeof(uvcb), + .handle = handle, + }; + int cc; + + WARN(!handle, "No handle provided to Ultravisor call cmd %x\n", cmd); + cc = uv_call_sched(0, (u64)&uvcb); + *rc = uvcb.header.rc; + *rrc = uvcb.header.rrc; + return cc ? -EINVAL : 0; +} + +struct uv_info { + unsigned long inst_calls_list[4]; + unsigned long uv_base_stor_len; + unsigned long guest_base_stor_len; + unsigned long guest_virt_base_stor_len; + unsigned long guest_virt_var_stor_len; + unsigned long guest_cpu_stor_len; + unsigned long max_sec_stor_addr; + unsigned int max_num_sec_conf; + unsigned short max_guest_cpus; +}; + +extern struct uv_info uv_info; + #ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST extern int prot_virt_guest; @@ -121,11 +329,40 @@ static inline int uv_remove_shared(unsigned long addr) return share(addr, UVC_CMD_REMOVE_SHARED_ACCESS); } -void uv_query_info(void); #else #define is_prot_virt_guest() 0 static inline int uv_set_shared(unsigned long addr) { return 0; } static inline int uv_remove_shared(unsigned long addr) { return 0; } +#endif + +#if IS_ENABLED(CONFIG_KVM) +extern int prot_virt_host; + +static inline int is_prot_virt_host(void) +{ + return prot_virt_host; +} + +int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb); +int uv_convert_from_secure(unsigned long paddr); +int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr); + +void setup_uv(void); +void adjust_to_uv_max(unsigned long *vmax); +#else +#define is_prot_virt_host() 0 +static inline void setup_uv(void) {} +static inline void adjust_to_uv_max(unsigned long *vmax) {} + +static inline int uv_convert_from_secure(unsigned long paddr) +{ + return 0; +} +#endif + +#if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) || IS_ENABLED(CONFIG_KVM) +void uv_query_info(void); +#else static inline void uv_query_info(void) {} #endif diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 2b1203cf7be6..33d4de233c5b 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -70,7 +70,7 @@ obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o -obj-$(CONFIG_IMA) += ima_arch.o +obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT) += ima_arch.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_cpum_cf_common.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf.o perf_cpum_sf.o @@ -78,6 +78,7 @@ obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf_events.o perf_regs.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf_diag.o obj-$(CONFIG_TRACEPOINTS) += trace.o +obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_PGSTE)) += uv.o # vdso obj-y += vdso64/ diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index 1d3927e01a5f..faca269d5f27 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -24,6 +24,8 @@ asmlinkage void do_syscall_trace_exit(struct pt_regs *regs); void do_protection_exception(struct pt_regs *regs); void do_dat_exception(struct pt_regs *regs); +void do_secure_storage_access(struct pt_regs *regs); +void do_non_secure_storage_access(struct pt_regs *regs); void addressing_exception(struct pt_regs *regs); void data_exception(struct pt_regs *regs); diff --git a/arch/s390/kernel/pgm_check.S b/arch/s390/kernel/pgm_check.S index eee3a482195a..2c27907a5ffc 100644 --- a/arch/s390/kernel/pgm_check.S +++ b/arch/s390/kernel/pgm_check.S @@ -78,8 +78,8 @@ PGM_CHECK(do_dat_exception) /* 39 */ PGM_CHECK(do_dat_exception) /* 3a */ PGM_CHECK(do_dat_exception) /* 3b */ PGM_CHECK_DEFAULT /* 3c */ -PGM_CHECK_DEFAULT /* 3d */ -PGM_CHECK_DEFAULT /* 3e */ +PGM_CHECK(do_secure_storage_access) /* 3d */ +PGM_CHECK(do_non_secure_storage_access) /* 3e */ PGM_CHECK_DEFAULT /* 3f */ PGM_CHECK(monitor_event_exception) /* 40 */ PGM_CHECK_DEFAULT /* 41 */ diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index b2c2f75860e8..1423090a2259 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -92,10 +92,6 @@ char elf_platform[ELF_PLATFORM_SIZE]; unsigned long int_hwcap = 0; -#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST -int __bootdata_preserved(prot_virt_guest); -#endif - int __bootdata(noexec_disabled); int __bootdata(memory_end_set); unsigned long __bootdata(memory_end); @@ -564,6 +560,9 @@ static void __init setup_memory_end(void) vmax = _REGION1_SIZE; /* 4-level kernel page table */ } + if (is_prot_virt_host()) + adjust_to_uv_max(&vmax); + /* module area is at the end of the kernel address space. */ MODULES_END = vmax; MODULES_VADDR = MODULES_END - MODULES_LEN; @@ -1138,6 +1137,8 @@ void __init setup_arch(char **cmdline_p) */ memblock_trim_memory(1UL << (MAX_ORDER - 1 + PAGE_SHIFT)); + if (is_prot_virt_host()) + setup_uv(); setup_memory_end(); setup_memory(); dma_contiguous_reserve(memory_end); diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c new file mode 100644 index 000000000000..c86d654351d1 --- /dev/null +++ b/arch/s390/kernel/uv.c @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Common Ultravisor functions and initialization + * + * Copyright IBM Corp. 2019, 2020 + */ +#define KMSG_COMPONENT "prot_virt" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* the bootdata_preserved fields come from ones in arch/s390/boot/uv.c */ +#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST +int __bootdata_preserved(prot_virt_guest); +#endif + +#if IS_ENABLED(CONFIG_KVM) +int prot_virt_host; +EXPORT_SYMBOL(prot_virt_host); +struct uv_info __bootdata_preserved(uv_info); +EXPORT_SYMBOL(uv_info); + +static int __init prot_virt_setup(char *val) +{ + bool enabled; + int rc; + + rc = kstrtobool(val, &enabled); + if (!rc && enabled) + prot_virt_host = 1; + + if (is_prot_virt_guest() && prot_virt_host) { + prot_virt_host = 0; + pr_warn("Protected virtualization not available in protected guests."); + } + + if (prot_virt_host && !test_facility(158)) { + prot_virt_host = 0; + pr_warn("Protected virtualization not supported by the hardware."); + } + + return rc; +} +early_param("prot_virt", prot_virt_setup); + +static int __init uv_init(unsigned long stor_base, unsigned long stor_len) +{ + struct uv_cb_init uvcb = { + .header.cmd = UVC_CMD_INIT_UV, + .header.len = sizeof(uvcb), + .stor_origin = stor_base, + .stor_len = stor_len, + }; + + if (uv_call(0, (uint64_t)&uvcb)) { + pr_err("Ultravisor init failed with rc: 0x%x rrc: 0%x\n", + uvcb.header.rc, uvcb.header.rrc); + return -1; + } + return 0; +} + +void __init setup_uv(void) +{ + unsigned long uv_stor_base; + + uv_stor_base = (unsigned long)memblock_alloc_try_nid( + uv_info.uv_base_stor_len, SZ_1M, SZ_2G, + MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE); + if (!uv_stor_base) { + pr_warn("Failed to reserve %lu bytes for ultravisor base storage\n", + uv_info.uv_base_stor_len); + goto fail; + } + + if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) { + memblock_free(uv_stor_base, uv_info.uv_base_stor_len); + goto fail; + } + + pr_info("Reserving %luMB as ultravisor base storage\n", + uv_info.uv_base_stor_len >> 20); + return; +fail: + pr_info("Disabling support for protected virtualization"); + prot_virt_host = 0; +} + +void adjust_to_uv_max(unsigned long *vmax) +{ + *vmax = min_t(unsigned long, *vmax, uv_info.max_sec_stor_addr); +} + +/* + * Requests the Ultravisor to pin the page in the shared state. This will + * cause an intercept when the guest attempts to unshare the pinned page. + */ +static int uv_pin_shared(unsigned long paddr) +{ + struct uv_cb_cfs uvcb = { + .header.cmd = UVC_CMD_PIN_PAGE_SHARED, + .header.len = sizeof(uvcb), + .paddr = paddr, + }; + + if (uv_call(0, (u64)&uvcb)) + return -EINVAL; + return 0; +} + +/* + * Requests the Ultravisor to encrypt a guest page and make it + * accessible to the host for paging (export). + * + * @paddr: Absolute host address of page to be exported + */ +int uv_convert_from_secure(unsigned long paddr) +{ + struct uv_cb_cfs uvcb = { + .header.cmd = UVC_CMD_CONV_FROM_SEC_STOR, + .header.len = sizeof(uvcb), + .paddr = paddr + }; + + if (uv_call(0, (u64)&uvcb)) + return -EINVAL; + return 0; +} + +/* + * Calculate the expected ref_count for a page that would otherwise have no + * further pins. This was cribbed from similar functions in other places in + * the kernel, but with some slight modifications. We know that a secure + * page can not be a huge page for example. + */ +static int expected_page_refs(struct page *page) +{ + int res; + + res = page_mapcount(page); + if (PageSwapCache(page)) { + res++; + } else if (page_mapping(page)) { + res++; + if (page_has_private(page)) + res++; + } + return res; +} + +static int make_secure_pte(pte_t *ptep, unsigned long addr, + struct page *exp_page, struct uv_cb_header *uvcb) +{ + pte_t entry = READ_ONCE(*ptep); + struct page *page; + int expected, rc = 0; + + if (!pte_present(entry)) + return -ENXIO; + if (pte_val(entry) & _PAGE_INVALID) + return -ENXIO; + + page = pte_page(entry); + if (page != exp_page) + return -ENXIO; + if (PageWriteback(page)) + return -EAGAIN; + expected = expected_page_refs(page); + if (!page_ref_freeze(page, expected)) + return -EBUSY; + set_bit(PG_arch_1, &page->flags); + rc = uv_call(0, (u64)uvcb); + page_ref_unfreeze(page, expected); + /* Return -ENXIO if the page was not mapped, -EINVAL otherwise */ + if (rc) + rc = uvcb->rc == 0x10a ? -ENXIO : -EINVAL; + return rc; +} + +/* + * Requests the Ultravisor to make a page accessible to a guest. + * If it's brought in the first time, it will be cleared. If + * it has been exported before, it will be decrypted and integrity + * checked. + */ +int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb) +{ + struct vm_area_struct *vma; + bool local_drain = false; + spinlock_t *ptelock; + unsigned long uaddr; + struct page *page; + pte_t *ptep; + int rc; + +again: + rc = -EFAULT; + down_read(&gmap->mm->mmap_sem); + + uaddr = __gmap_translate(gmap, gaddr); + if (IS_ERR_VALUE(uaddr)) + goto out; + vma = find_vma(gmap->mm, uaddr); + if (!vma) + goto out; + /* + * Secure pages cannot be huge and userspace should not combine both. + * In case userspace does it anyway this will result in an -EFAULT for + * the unpack. The guest is thus never reaching secure mode. If + * userspace is playing dirty tricky with mapping huge pages later + * on this will result in a segmentation fault. + */ + if (is_vm_hugetlb_page(vma)) + goto out; + + rc = -ENXIO; + page = follow_page(vma, uaddr, FOLL_WRITE); + if (IS_ERR_OR_NULL(page)) + goto out; + + lock_page(page); + ptep = get_locked_pte(gmap->mm, uaddr, &ptelock); + rc = make_secure_pte(ptep, uaddr, page, uvcb); + pte_unmap_unlock(ptep, ptelock); + unlock_page(page); +out: + up_read(&gmap->mm->mmap_sem); + + if (rc == -EAGAIN) { + wait_on_page_writeback(page); + } else if (rc == -EBUSY) { + /* + * If we have tried a local drain and the page refcount + * still does not match our expected safe value, try with a + * system wide drain. This is needed if the pagevecs holding + * the page are on a different CPU. + */ + if (local_drain) { + lru_add_drain_all(); + /* We give up here, and let the caller try again */ + return -EAGAIN; + } + /* + * We are here if the page refcount does not match the + * expected safe value. The main culprits are usually + * pagevecs. With lru_add_drain() we drain the pagevecs + * on the local CPU so that hopefully the refcount will + * reach the expected safe value. + */ + lru_add_drain(); + local_drain = true; + /* And now we try again immediately after draining */ + goto again; + } else if (rc == -ENXIO) { + if (gmap_fault(gmap, gaddr, FAULT_FLAG_WRITE)) + return -EFAULT; + return -EAGAIN; + } + return rc; +} +EXPORT_SYMBOL_GPL(gmap_make_secure); + +int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr) +{ + struct uv_cb_cts uvcb = { + .header.cmd = UVC_CMD_CONV_TO_SEC_STOR, + .header.len = sizeof(uvcb), + .guest_handle = gmap->guest_handle, + .gaddr = gaddr, + }; + + return gmap_make_secure(gmap, gaddr, &uvcb); +} +EXPORT_SYMBOL_GPL(gmap_convert_to_secure); + +/* + * To be called with the page locked or with an extra reference! This will + * prevent gmap_make_secure from touching the page concurrently. Having 2 + * parallel make_page_accessible is fine, as the UV calls will become a + * no-op if the page is already exported. + */ +int arch_make_page_accessible(struct page *page) +{ + int rc = 0; + + /* Hugepage cannot be protected, so nothing to do */ + if (PageHuge(page)) + return 0; + + /* + * PG_arch_1 is used in 3 places: + * 1. for kernel page tables during early boot + * 2. for storage keys of huge pages and KVM + * 3. As an indication that this page might be secure. This can + * overindicate, e.g. we set the bit before calling + * convert_to_secure. + * As secure pages are never huge, all 3 variants can co-exists. + */ + if (!test_bit(PG_arch_1, &page->flags)) + return 0; + + rc = uv_pin_shared(page_to_phys(page)); + if (!rc) { + clear_bit(PG_arch_1, &page->flags); + return 0; + } + + rc = uv_convert_from_secure(page_to_phys(page)); + if (!rc) { + clear_bit(PG_arch_1, &page->flags); + return 0; + } + + return rc; +} +EXPORT_SYMBOL_GPL(arch_make_page_accessible); + +#endif + +#if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) || IS_ENABLED(CONFIG_KVM) +static ssize_t uv_query_facilities(struct kobject *kobj, + struct kobj_attribute *attr, char *page) +{ + return snprintf(page, PAGE_SIZE, "%lx\n%lx\n%lx\n%lx\n", + uv_info.inst_calls_list[0], + uv_info.inst_calls_list[1], + uv_info.inst_calls_list[2], + uv_info.inst_calls_list[3]); +} + +static struct kobj_attribute uv_query_facilities_attr = + __ATTR(facilities, 0444, uv_query_facilities, NULL); + +static ssize_t uv_query_max_guest_cpus(struct kobject *kobj, + struct kobj_attribute *attr, char *page) +{ + return snprintf(page, PAGE_SIZE, "%d\n", + uv_info.max_guest_cpus); +} + +static struct kobj_attribute uv_query_max_guest_cpus_attr = + __ATTR(max_cpus, 0444, uv_query_max_guest_cpus, NULL); + +static ssize_t uv_query_max_guest_vms(struct kobject *kobj, + struct kobj_attribute *attr, char *page) +{ + return snprintf(page, PAGE_SIZE, "%d\n", + uv_info.max_num_sec_conf); +} + +static struct kobj_attribute uv_query_max_guest_vms_attr = + __ATTR(max_guests, 0444, uv_query_max_guest_vms, NULL); + +static ssize_t uv_query_max_guest_addr(struct kobject *kobj, + struct kobj_attribute *attr, char *page) +{ + return snprintf(page, PAGE_SIZE, "%lx\n", + uv_info.max_sec_stor_addr); +} + +static struct kobj_attribute uv_query_max_guest_addr_attr = + __ATTR(max_address, 0444, uv_query_max_guest_addr, NULL); + +static struct attribute *uv_query_attrs[] = { + &uv_query_facilities_attr.attr, + &uv_query_max_guest_cpus_attr.attr, + &uv_query_max_guest_vms_attr.attr, + &uv_query_max_guest_addr_attr.attr, + NULL, +}; + +static struct attribute_group uv_query_attr_group = { + .attrs = uv_query_attrs, +}; + +static struct kset *uv_query_kset; +static struct kobject *uv_kobj; + +static int __init uv_info_init(void) +{ + int rc = -ENOMEM; + + if (!test_facility(158)) + return 0; + + uv_kobj = kobject_create_and_add("uv", firmware_kobj); + if (!uv_kobj) + return -ENOMEM; + + uv_query_kset = kset_create_and_add("query", NULL, uv_kobj); + if (!uv_query_kset) + goto out_kobj; + + rc = sysfs_create_group(&uv_query_kset->kobj, &uv_query_attr_group); + if (!rc) + return 0; + + kset_unregister(uv_query_kset); +out_kobj: + kobject_del(uv_kobj); + kobject_put(uv_kobj); + return rc; +} +device_initcall(uv_info_init); +#endif diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index 05ee90a5ea08..12decca22e7c 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -9,6 +9,6 @@ common-objs = $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/async_pf.o $(KVM)/irqch ccflags-y := -Ivirt/kvm -Iarch/s390/kvm kvm-objs := $(common-objs) kvm-s390.o intercept.o interrupt.o priv.o sigp.o -kvm-objs += diag.o gaccess.o guestdbg.o vsie.o +kvm-objs += diag.o gaccess.o guestdbg.o vsie.o pv.o obj-$(CONFIG_KVM) += kvm.o diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c index 3fb54ec2cf3e..563429dece03 100644 --- a/arch/s390/kvm/diag.c +++ b/arch/s390/kvm/diag.c @@ -2,7 +2,7 @@ /* * handling diagnose instructions * - * Copyright IBM Corp. 2008, 2011 + * Copyright IBM Corp. 2008, 2020 * * Author(s): Carsten Otte * Christian Borntraeger @@ -201,6 +201,10 @@ static int __diag_ipl_functions(struct kvm_vcpu *vcpu) return -EOPNOTSUPP; } + /* + * no need to check the return value of vcpu_stop as it can only have + * an error for protvirt, but protvirt means user cpu state + */ if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) kvm_s390_vcpu_stop(vcpu); vcpu->run->s390_reset_flags |= KVM_S390_RESET_SUBSYSTEM; diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 07d30ffcfa41..47a67a958107 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -505,7 +505,7 @@ static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, switch (prot) { case PROT_TYPE_IEP: tec->b61 = 1; - /* FALL THROUGH */ + fallthrough; case PROT_TYPE_LA: tec->b56 = 1; break; @@ -514,12 +514,12 @@ static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, break; case PROT_TYPE_ALC: tec->b60 = 1; - /* FALL THROUGH */ + fallthrough; case PROT_TYPE_DAT: tec->b61 = 1; break; } - /* FALL THROUGH */ + fallthrough; case PGM_ASCE_TYPE: case PGM_PAGE_TRANSLATION: case PGM_REGION_FIRST_TRANS: @@ -534,7 +534,7 @@ static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, tec->addr = gva >> PAGE_SHIFT; tec->fsi = mode == GACC_STORE ? FSI_STORE : FSI_FETCH; tec->as = psw_bits(vcpu->arch.sie_block->gpsw).as; - /* FALL THROUGH */ + fallthrough; case PGM_ALEN_TRANSLATION: case PGM_ALE_SEQUENCE: case PGM_ASTE_VALIDITY: @@ -677,7 +677,7 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, dat_protection |= rfte.p; ptr = rfte.rto * PAGE_SIZE + vaddr.rsx * 8; } - /* fallthrough */ + fallthrough; case ASCE_TYPE_REGION2: { union region2_table_entry rste; @@ -695,7 +695,7 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, dat_protection |= rste.p; ptr = rste.rto * PAGE_SIZE + vaddr.rtx * 8; } - /* fallthrough */ + fallthrough; case ASCE_TYPE_REGION3: { union region3_table_entry rtte; @@ -723,7 +723,7 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva, dat_protection |= rtte.fc0.p; ptr = rtte.fc0.sto * PAGE_SIZE + vaddr.sx * 8; } - /* fallthrough */ + fallthrough; case ASCE_TYPE_SEGMENT: { union segment_table_entry ste; @@ -1050,7 +1050,8 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake); if (rc) return rc; - } /* fallthrough */ + } + fallthrough; case ASCE_TYPE_REGION2: { union region2_table_entry rste; @@ -1076,7 +1077,8 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake); if (rc) return rc; - } /* fallthrough */ + } + fallthrough; case ASCE_TYPE_REGION3: { union region3_table_entry rtte; @@ -1111,7 +1113,8 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr, rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake); if (rc) return rc; - } /* fallthrough */ + } + fallthrough; case ASCE_TYPE_SEGMENT: { union segment_table_entry ste; diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c index a389fa85cca2..e7a7c499a73f 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/intercept.c @@ -2,7 +2,7 @@ /* * in-kernel handling for sie intercepts * - * Copyright IBM Corp. 2008, 2014 + * Copyright IBM Corp. 2008, 2020 * * Author(s): Carsten Otte * Christian Borntraeger @@ -12,10 +12,10 @@ #include #include -#include #include #include #include +#include #include "kvm-s390.h" #include "gaccess.h" @@ -79,6 +79,10 @@ static int handle_stop(struct kvm_vcpu *vcpu) return rc; } + /* + * no need to check the return value of vcpu_stop as it can only have + * an error for protvirt, but protvirt means user cpu state + */ if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) kvm_s390_vcpu_stop(vcpu); return -EOPNOTSUPP; @@ -231,6 +235,13 @@ static int handle_prog(struct kvm_vcpu *vcpu) vcpu->stat.exit_program_interruption++; + /* + * Intercept 8 indicates a loop of specification exceptions + * for protected guests. + */ + if (kvm_s390_pv_cpu_is_protected(vcpu)) + return -EOPNOTSUPP; + if (guestdbg_enabled(vcpu) && per_event(vcpu)) { rc = kvm_s390_handle_per_event(vcpu); if (rc) @@ -384,7 +395,7 @@ int handle_sthyi(struct kvm_vcpu *vcpu) goto out; } - if (addr & ~PAGE_MASK) + if (!kvm_s390_pv_cpu_is_protected(vcpu) && (addr & ~PAGE_MASK)) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); sctns = (void *)get_zeroed_page(GFP_KERNEL); @@ -395,10 +406,15 @@ int handle_sthyi(struct kvm_vcpu *vcpu) out: if (!cc) { - r = write_guest(vcpu, addr, reg2, sctns, PAGE_SIZE); - if (r) { - free_page((unsigned long)sctns); - return kvm_s390_inject_prog_cond(vcpu, r); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + memcpy((void *)(sida_origin(vcpu->arch.sie_block)), + sctns, PAGE_SIZE); + } else { + r = write_guest(vcpu, addr, reg2, sctns, PAGE_SIZE); + if (r) { + free_page((unsigned long)sctns); + return kvm_s390_inject_prog_cond(vcpu, r); + } } } @@ -444,6 +460,77 @@ static int handle_operexc(struct kvm_vcpu *vcpu) return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); } +static int handle_pv_spx(struct kvm_vcpu *vcpu) +{ + u32 pref = *(u32 *)vcpu->arch.sie_block->sidad; + + kvm_s390_set_prefix(vcpu, pref); + trace_kvm_s390_handle_prefix(vcpu, 1, pref); + return 0; +} + +static int handle_pv_sclp(struct kvm_vcpu *vcpu) +{ + struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; + + spin_lock(&fi->lock); + /* + * 2 cases: + * a: an sccb answering interrupt was already pending or in flight. + * As the sccb value is not known we can simply set some value to + * trigger delivery of a saved SCCB. UV will then use its saved + * copy of the SCCB value. + * b: an error SCCB interrupt needs to be injected so we also inject + * a fake SCCB address. Firmware will use the proper one. + * This makes sure, that both errors and real sccb returns will only + * be delivered after a notification intercept (instruction has + * finished) but not after others. + */ + fi->srv_signal.ext_params |= 0x43000; + set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); + clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); + spin_unlock(&fi->lock); + return 0; +} + +static int handle_pv_uvc(struct kvm_vcpu *vcpu) +{ + struct uv_cb_share *guest_uvcb = (void *)vcpu->arch.sie_block->sidad; + struct uv_cb_cts uvcb = { + .header.cmd = UVC_CMD_UNPIN_PAGE_SHARED, + .header.len = sizeof(uvcb), + .guest_handle = kvm_s390_pv_get_handle(vcpu->kvm), + .gaddr = guest_uvcb->paddr, + }; + int rc; + + if (guest_uvcb->header.cmd != UVC_CMD_REMOVE_SHARED_ACCESS) { + WARN_ONCE(1, "Unexpected notification intercept for UVC 0x%x\n", + guest_uvcb->header.cmd); + return 0; + } + rc = gmap_make_secure(vcpu->arch.gmap, uvcb.gaddr, &uvcb); + /* + * If the unpin did not succeed, the guest will exit again for the UVC + * and we will retry the unpin. + */ + if (rc == -EINVAL) + return 0; + return rc; +} + +static int handle_pv_notification(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.sie_block->ipa == 0xb210) + return handle_pv_spx(vcpu); + if (vcpu->arch.sie_block->ipa == 0xb220) + return handle_pv_sclp(vcpu); + if (vcpu->arch.sie_block->ipa == 0xb9a4) + return handle_pv_uvc(vcpu); + + return handle_instruction(vcpu); +} + int kvm_handle_sie_intercept(struct kvm_vcpu *vcpu) { int rc, per_rc = 0; @@ -480,6 +567,28 @@ int kvm_handle_sie_intercept(struct kvm_vcpu *vcpu) case ICPT_KSS: rc = kvm_s390_skey_check_enable(vcpu); break; + case ICPT_MCHKREQ: + case ICPT_INT_ENABLE: + /* + * PSW bit 13 or a CR (0, 6, 14) changed and we might + * now be able to deliver interrupts. The pre-run code + * will take care of this. + */ + rc = 0; + break; + case ICPT_PV_INSTR: + rc = handle_instruction(vcpu); + break; + case ICPT_PV_NOTIFY: + rc = handle_pv_notification(vcpu); + break; + case ICPT_PV_PREF: + rc = 0; + gmap_convert_to_secure(vcpu->arch.gmap, + kvm_s390_get_prefix(vcpu)); + gmap_convert_to_secure(vcpu->arch.gmap, + kvm_s390_get_prefix(vcpu) + PAGE_SIZE); + break; default: return -EOPNOTSUPP; } diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index c06c89d370a7..8191106bf7b9 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -2,7 +2,7 @@ /* * handling kvm guest interrupts * - * Copyright IBM Corp. 2008, 2015 + * Copyright IBM Corp. 2008, 2020 * * Author(s): Carsten Otte */ @@ -324,8 +324,11 @@ static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu) { - return vcpu->kvm->arch.float_int.pending_irqs | - vcpu->arch.local_int.pending_irqs; + unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs | + vcpu->arch.local_int.pending_irqs; + + pending &= ~vcpu->kvm->arch.float_int.masked_irqs; + return pending; } static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu) @@ -383,10 +386,18 @@ static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask); if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK)) __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask); - if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) + if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) { __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask); + __clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask); + } if (psw_mchk_disabled(vcpu)) active_mask &= ~IRQ_PEND_MCHK_MASK; + /* PV guest cpus can have a single interruption injected at a time. */ + if (kvm_s390_pv_cpu_is_protected(vcpu) && + vcpu->arch.sie_block->iictl != IICTL_CODE_NONE) + active_mask &= ~(IRQ_PEND_EXT_II_MASK | + IRQ_PEND_IO_MASK | + IRQ_PEND_MCHK_MASK); /* * Check both floating and local interrupt's cr14 because * bit IRQ_PEND_MCHK_REP could be set in both cases. @@ -479,19 +490,23 @@ static void set_intercept_indicators(struct kvm_vcpu *vcpu) static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; - int rc; + int rc = 0; vcpu->stat.deliver_cputm++; trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 0, 0); - - rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER, - (u16 *)__LC_EXT_INT_CODE); - rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); - rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, - &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); - rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, - &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; + vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER; + } else { + rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER, + (u16 *)__LC_EXT_INT_CODE); + rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); + rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, + &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, + &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + } clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); return rc ? -EFAULT : 0; } @@ -499,19 +514,23 @@ static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu) static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; - int rc; + int rc = 0; vcpu->stat.deliver_ckc++; trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 0, 0); - - rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP, - (u16 __user *)__LC_EXT_INT_CODE); - rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); - rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, - &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); - rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, - &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; + vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP; + } else { + rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP, + (u16 __user *)__LC_EXT_INT_CODE); + rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); + rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, + &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, + &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + } clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); return rc ? -EFAULT : 0; } @@ -553,6 +572,20 @@ static int __write_machine_check(struct kvm_vcpu *vcpu, union mci mci; int rc; + /* + * All other possible payload for a machine check (e.g. the register + * contents in the save area) will be handled by the ultravisor, as + * the hypervisor does not not have the needed information for + * protected guests. + */ + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK; + vcpu->arch.sie_block->mcic = mchk->mcic; + vcpu->arch.sie_block->faddr = mchk->failing_storage_address; + vcpu->arch.sie_block->edc = mchk->ext_damage_code; + return 0; + } + mci.val = mchk->mcic; /* take care of lazy register loading */ save_fpu_regs(); @@ -696,17 +729,21 @@ static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) static int __must_check __deliver_restart(struct kvm_vcpu *vcpu) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; - int rc; + int rc = 0; VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart"); vcpu->stat.deliver_restart_signal++; trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); - rc = write_guest_lc(vcpu, - offsetof(struct lowcore, restart_old_psw), - &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); - rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw), - &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART; + } else { + rc = write_guest_lc(vcpu, + offsetof(struct lowcore, restart_old_psw), + &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw), + &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + } clear_bit(IRQ_PEND_RESTART, &li->pending_irqs); return rc ? -EFAULT : 0; } @@ -748,6 +785,12 @@ static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu) vcpu->stat.deliver_emergency_signal++; trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, cpu_addr, 0); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; + vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG; + vcpu->arch.sie_block->extcpuaddr = cpu_addr; + return 0; + } rc = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG, (u16 *)__LC_EXT_INT_CODE); @@ -776,6 +819,12 @@ static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu) trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL, extcall.code, 0); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; + vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL; + vcpu->arch.sie_block->extcpuaddr = extcall.code; + return 0; + } rc = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL, (u16 *)__LC_EXT_INT_CODE); @@ -787,6 +836,21 @@ static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu) return rc ? -EFAULT : 0; } +static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code) +{ + switch (code) { + case PGM_SPECIFICATION: + vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION; + break; + case PGM_OPERAND: + vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND; + break; + default: + return -EINVAL; + } + return 0; +} + static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; @@ -807,6 +871,10 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, pgm_info.code, 0); + /* PER is handled by the ultravisor */ + if (kvm_s390_pv_cpu_is_protected(vcpu)) + return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER); + switch (pgm_info.code & ~PGM_PER) { case PGM_AFX_TRANSLATION: case PGM_ASX_TRANSLATION: @@ -818,7 +886,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) case PGM_PRIMARY_AUTHORITY: case PGM_SECONDARY_AUTHORITY: nullifying = true; - /* fall through */ + fallthrough; case PGM_SPACE_SWITCH: rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, (u64 *)__LC_TRANS_EXC_CODE); @@ -902,20 +970,49 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) return rc ? -EFAULT : 0; } +#define SCCB_MASK 0xFFFFFFF8 +#define SCCB_EVENT_PENDING 0x3 + +static int write_sclp(struct kvm_vcpu *vcpu, u32 parm) +{ + int rc; + + if (kvm_s390_pv_cpu_get_handle(vcpu)) { + vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; + vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG; + vcpu->arch.sie_block->eiparams = parm; + return 0; + } + + rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); + rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); + rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, + &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, + &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); + rc |= put_guest_lc(vcpu, parm, + (u32 *)__LC_EXT_PARAMS); + + return rc ? -EFAULT : 0; +} + static int __must_check __deliver_service(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_ext_info ext; - int rc = 0; spin_lock(&fi->lock); - if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { + if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) || + !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { spin_unlock(&fi->lock); return 0; } ext = fi->srv_signal; memset(&fi->srv_signal, 0, sizeof(ext)); clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); + clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); + if (kvm_s390_pv_cpu_is_protected(vcpu)) + set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); spin_unlock(&fi->lock); VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x", @@ -924,16 +1021,31 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu) trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, ext.ext_params, 0); - rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); - rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); - rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, - &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); - rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, - &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); - rc |= put_guest_lc(vcpu, ext.ext_params, - (u32 *)__LC_EXT_PARAMS); + return write_sclp(vcpu, ext.ext_params); +} - return rc ? -EFAULT : 0; +static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu) +{ + struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; + struct kvm_s390_ext_info ext; + + spin_lock(&fi->lock); + if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) { + spin_unlock(&fi->lock); + return 0; + } + ext = fi->srv_signal; + /* only clear the event bit */ + fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING; + clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); + spin_unlock(&fi->lock); + + VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event"); + vcpu->stat.deliver_service_signal++; + trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, + ext.ext_params, 0); + + return write_sclp(vcpu, SCCB_EVENT_PENDING); } static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) @@ -1028,6 +1140,15 @@ static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io) { int rc; + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + vcpu->arch.sie_block->iictl = IICTL_CODE_IO; + vcpu->arch.sie_block->subchannel_id = io->subchannel_id; + vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr; + vcpu->arch.sie_block->io_int_parm = io->io_int_parm; + vcpu->arch.sie_block->io_int_word = io->io_int_word; + return 0; + } + rc = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID); rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR); rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM); @@ -1329,6 +1450,9 @@ int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu) case IRQ_PEND_EXT_SERVICE: rc = __deliver_service(vcpu); break; + case IRQ_PEND_EXT_SERVICE_EV: + rc = __deliver_service_ev(vcpu); + break; case IRQ_PEND_PFAULT_DONE: rc = __deliver_pfault_done(vcpu); break; @@ -1421,7 +1545,7 @@ static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL) return -EINVAL; - if (sclp.has_sigpif) + if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu)) return sca_inject_ext_call(vcpu, src_id); if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) @@ -1681,9 +1805,6 @@ struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, return inti; } -#define SCCB_MASK 0xFFFFFFF8 -#define SCCB_EVENT_PENDING 0x3 - static int __inject_service(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { @@ -1692,6 +1813,11 @@ static int __inject_service(struct kvm *kvm, kvm->stat.inject_service_signal++; spin_lock(&fi->lock); fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; + + /* We always allow events, track them separately from the sccb ints */ + if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING) + set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); + /* * Early versions of the QEMU s390 bios will inject several * service interrupts after another without handling a @@ -1773,7 +1899,14 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) kvm->stat.inject_io++; isc = int_word_to_isc(inti->io.io_int_word); - if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) { + /* + * Do not make use of gisa in protected mode. We do not use the lock + * checking variant as this is just a performance optimization and we + * do not hold the lock here. This is ok as the code will pick + * interrupts from both "lists" for delivery. + */ + if (!kvm_s390_pv_get_handle(kvm) && + gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) { VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc); gisa_set_ipm_gisc(gi->origin, isc); kfree(inti); @@ -1834,7 +1967,8 @@ static void __floating_irq_kick(struct kvm *kvm, u64 type) break; case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: if (!(type & KVM_S390_INT_IO_AI_MASK && - kvm->arch.gisa_int.origin)) + kvm->arch.gisa_int.origin) || + kvm_s390_pv_cpu_get_handle(dst_vcpu)) kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT); break; default: @@ -2080,6 +2214,10 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; int i; + mutex_lock(&kvm->lock); + if (!kvm_s390_pv_is_protected(kvm)) + fi->masked_irqs = 0; + mutex_unlock(&kvm->lock); spin_lock(&fi->lock); fi->pending_irqs = 0; memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); @@ -2146,7 +2284,8 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) n++; } } - if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) { + if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) || + test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) { if (n == max_irqs) { /* signal userspace to try again */ ret = -ENOMEM; @@ -2327,9 +2466,6 @@ static int register_io_adapter(struct kvm_device *dev, if (!adapter) return -ENOMEM; - INIT_LIST_HEAD(&adapter->maps); - init_rwsem(&adapter->maps_lock); - atomic_set(&adapter->nr_maps, 0); adapter->id = adapter_info.id; adapter->isc = adapter_info.isc; adapter->maskable = adapter_info.maskable; @@ -2354,87 +2490,12 @@ int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) return ret; } -static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) -{ - struct s390_io_adapter *adapter = get_io_adapter(kvm, id); - struct s390_map_info *map; - int ret; - - if (!adapter || !addr) - return -EINVAL; - - map = kzalloc(sizeof(*map), GFP_KERNEL); - if (!map) { - ret = -ENOMEM; - goto out; - } - INIT_LIST_HEAD(&map->list); - map->guest_addr = addr; - map->addr = gmap_translate(kvm->arch.gmap, addr); - if (map->addr == -EFAULT) { - ret = -EFAULT; - goto out; - } - ret = get_user_pages_fast(map->addr, 1, FOLL_WRITE, &map->page); - if (ret < 0) - goto out; - BUG_ON(ret != 1); - down_write(&adapter->maps_lock); - if (atomic_inc_return(&adapter->nr_maps) < MAX_S390_ADAPTER_MAPS) { - list_add_tail(&map->list, &adapter->maps); - ret = 0; - } else { - put_page(map->page); - ret = -EINVAL; - } - up_write(&adapter->maps_lock); -out: - if (ret) - kfree(map); - return ret; -} - -static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) -{ - struct s390_io_adapter *adapter = get_io_adapter(kvm, id); - struct s390_map_info *map, *tmp; - int found = 0; - - if (!adapter || !addr) - return -EINVAL; - - down_write(&adapter->maps_lock); - list_for_each_entry_safe(map, tmp, &adapter->maps, list) { - if (map->guest_addr == addr) { - found = 1; - atomic_dec(&adapter->nr_maps); - list_del(&map->list); - put_page(map->page); - kfree(map); - break; - } - } - up_write(&adapter->maps_lock); - - return found ? 0 : -EINVAL; -} - void kvm_s390_destroy_adapters(struct kvm *kvm) { int i; - struct s390_map_info *map, *tmp; - for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { - if (!kvm->arch.adapters[i]) - continue; - list_for_each_entry_safe(map, tmp, - &kvm->arch.adapters[i]->maps, list) { - list_del(&map->list); - put_page(map->page); - kfree(map); - } + for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) kfree(kvm->arch.adapters[i]); - } } static int modify_io_adapter(struct kvm_device *dev, @@ -2456,11 +2517,14 @@ static int modify_io_adapter(struct kvm_device *dev, if (ret > 0) ret = 0; break; + /* + * The following operations are no longer needed and therefore no-ops. + * The gpa to hva translation is done when an IRQ route is set up. The + * set_irq code uses get_user_pages_remote() to do the actual write. + */ case KVM_S390_IO_ADAPTER_MAP: - ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr); - break; case KVM_S390_IO_ADAPTER_UNMAP: - ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr); + ret = 0; break; default: ret = -EINVAL; @@ -2699,19 +2763,15 @@ static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; } -static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, - u64 addr) +static struct page *get_map_page(struct kvm *kvm, u64 uaddr) { - struct s390_map_info *map; + struct page *page = NULL; - if (!adapter) - return NULL; - - list_for_each_entry(map, &adapter->maps, list) { - if (map->guest_addr == addr) - return map; - } - return NULL; + down_read(&kvm->mm->mmap_sem); + get_user_pages_remote(NULL, kvm->mm, uaddr, 1, FOLL_WRITE, + &page, NULL, NULL); + up_read(&kvm->mm->mmap_sem); + return page; } static int adapter_indicators_set(struct kvm *kvm, @@ -2720,30 +2780,35 @@ static int adapter_indicators_set(struct kvm *kvm, { unsigned long bit; int summary_set, idx; - struct s390_map_info *info; + struct page *ind_page, *summary_page; void *map; - info = get_map_info(adapter, adapter_int->ind_addr); - if (!info) + ind_page = get_map_page(kvm, adapter_int->ind_addr); + if (!ind_page) return -1; - map = page_address(info->page); - bit = get_ind_bit(info->addr, adapter_int->ind_offset, adapter->swap); - set_bit(bit, map); - idx = srcu_read_lock(&kvm->srcu); - mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); - set_page_dirty_lock(info->page); - info = get_map_info(adapter, adapter_int->summary_addr); - if (!info) { - srcu_read_unlock(&kvm->srcu, idx); + summary_page = get_map_page(kvm, adapter_int->summary_addr); + if (!summary_page) { + put_page(ind_page); return -1; } - map = page_address(info->page); - bit = get_ind_bit(info->addr, adapter_int->summary_offset, - adapter->swap); + + idx = srcu_read_lock(&kvm->srcu); + map = page_address(ind_page); + bit = get_ind_bit(adapter_int->ind_addr, + adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT); + set_page_dirty_lock(ind_page); + map = page_address(summary_page); + bit = get_ind_bit(adapter_int->summary_addr, + adapter_int->summary_offset, adapter->swap); summary_set = test_and_set_bit(bit, map); - mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); - set_page_dirty_lock(info->page); + mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT); + set_page_dirty_lock(summary_page); srcu_read_unlock(&kvm->srcu, idx); + + put_page(ind_page); + put_page(summary_page); return summary_set ? 0 : 1; } @@ -2765,9 +2830,7 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, adapter = get_io_adapter(kvm, e->adapter.adapter_id); if (!adapter) return -1; - down_read(&adapter->maps_lock); ret = adapter_indicators_set(kvm, adapter, &e->adapter); - up_read(&adapter->maps_lock); if ((ret > 0) && !adapter->masked) { ret = kvm_s390_inject_airq(kvm, adapter); if (ret == 0) @@ -2818,23 +2881,27 @@ int kvm_set_routing_entry(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, const struct kvm_irq_routing_entry *ue) { - int ret; + u64 uaddr; switch (ue->type) { + /* we store the userspace addresses instead of the guest addresses */ case KVM_IRQ_ROUTING_S390_ADAPTER: e->set = set_adapter_int; - e->adapter.summary_addr = ue->u.adapter.summary_addr; - e->adapter.ind_addr = ue->u.adapter.ind_addr; + uaddr = gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr); + if (uaddr == -EFAULT) + return -EFAULT; + e->adapter.summary_addr = uaddr; + uaddr = gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr); + if (uaddr == -EFAULT) + return -EFAULT; + e->adapter.ind_addr = uaddr; e->adapter.summary_offset = ue->u.adapter.summary_offset; e->adapter.ind_offset = ue->u.adapter.ind_offset; e->adapter.adapter_id = ue->u.adapter.adapter_id; - ret = 0; - break; + return 0; default: - ret = -EINVAL; + return -EINVAL; } - - return ret; } int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index c2e6d4ba4e23..19a81024fe16 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -2,7 +2,7 @@ /* * hosting IBM Z kernel virtual machines (s390x) * - * Copyright IBM Corp. 2008, 2018 + * Copyright IBM Corp. 2008, 2020 * * Author(s): Carsten Otte * Christian Borntraeger @@ -44,6 +44,7 @@ #include #include #include +#include #include "kvm-s390.h" #include "gaccess.h" @@ -184,6 +185,11 @@ static u8 halt_poll_max_steal = 10; module_param(halt_poll_max_steal, byte, 0644); MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling"); +/* if set to true, the GISA will be initialized and used if available */ +static bool use_gisa = true; +module_param(use_gisa, bool, 0644); +MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it."); + /* * For now we handle at most 16 double words as this is what the s390 base * kernel handles and stores in the prefix page. If we ever need to go beyond @@ -220,6 +226,7 @@ static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc; static struct gmap_notifier gmap_notifier; static struct gmap_notifier vsie_gmap_notifier; debug_info_t *kvm_s390_dbf; +debug_info_t *kvm_s390_dbf_uv; /* Section: not file related */ int kvm_arch_hardware_enable(void) @@ -228,13 +235,15 @@ int kvm_arch_hardware_enable(void) return 0; } -int kvm_arch_check_processor_compat(void) +int kvm_arch_check_processor_compat(void *opaque) { return 0; } +/* forward declarations */ static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start, unsigned long end); +static int sca_switch_to_extended(struct kvm *kvm); static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta) { @@ -293,7 +302,7 @@ static struct notifier_block kvm_clock_notifier = { .notifier_call = kvm_clock_sync, }; -int kvm_arch_hardware_setup(void) +int kvm_arch_hardware_setup(void *opaque) { gmap_notifier.notifier_call = kvm_gmap_notifier; gmap_register_pte_notifier(&gmap_notifier); @@ -460,7 +469,12 @@ int kvm_arch_init(void *opaque) if (!kvm_s390_dbf) return -ENOMEM; - if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) + kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long)); + if (!kvm_s390_dbf_uv) + goto out; + + if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) || + debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view)) goto out; kvm_s390_cpu_feat_init(); @@ -487,6 +501,7 @@ void kvm_arch_exit(void) { kvm_s390_gib_destroy(); debug_unregister(kvm_s390_dbf); + debug_unregister(kvm_s390_dbf_uv); } /* Section: device related */ @@ -564,14 +579,16 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_S390_BPB: r = test_facility(82); break; + case KVM_CAP_S390_PROTECTED: + r = is_prot_virt_host(); + break; default: r = 0; } return r; } -static void kvm_s390_sync_dirty_log(struct kvm *kvm, - struct kvm_memory_slot *memslot) +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { int i; gfn_t cur_gfn, last_gfn; @@ -612,9 +629,8 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, { int r; unsigned long n; - struct kvm_memslots *slots; struct kvm_memory_slot *memslot; - int is_dirty = 0; + int is_dirty; if (kvm_is_ucontrol(kvm)) return -EINVAL; @@ -625,14 +641,7 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, if (log->slot >= KVM_USER_MEM_SLOTS) goto out; - slots = kvm_memslots(kvm); - memslot = id_to_memslot(slots, log->slot); - r = -ENOENT; - if (!memslot->dirty_bitmap) - goto out; - - kvm_s390_sync_dirty_log(kvm, memslot); - r = kvm_get_dirty_log(kvm, log, &is_dirty); + r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot); if (r) goto out; @@ -1993,6 +2002,9 @@ static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args, struct kvm_memslots *slots = kvm_memslots(kvm); struct kvm_memory_slot *ms; + if (unlikely(!slots->used_slots)) + return 0; + cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn); ms = gfn_to_memslot(kvm, cur_gfn); args->count = 0; @@ -2158,6 +2170,194 @@ static int kvm_s390_set_cmma_bits(struct kvm *kvm, return r; } +static int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rcp, u16 *rrcp) +{ + struct kvm_vcpu *vcpu; + u16 rc, rrc; + int ret = 0; + int i; + + /* + * We ignore failures and try to destroy as many CPUs as possible. + * At the same time we must not free the assigned resources when + * this fails, as the ultravisor has still access to that memory. + * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak + * behind. + * We want to return the first failure rc and rrc, though. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + mutex_lock(&vcpu->mutex); + if (kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc) && !ret) { + *rcp = rc; + *rrcp = rrc; + ret = -EIO; + } + mutex_unlock(&vcpu->mutex); + } + return ret; +} + +static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc) +{ + int i, r = 0; + u16 dummy; + + struct kvm_vcpu *vcpu; + + kvm_for_each_vcpu(i, vcpu, kvm) { + mutex_lock(&vcpu->mutex); + r = kvm_s390_pv_create_cpu(vcpu, rc, rrc); + mutex_unlock(&vcpu->mutex); + if (r) + break; + } + if (r) + kvm_s390_cpus_from_pv(kvm, &dummy, &dummy); + return r; +} + +static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd) +{ + int r = 0; + u16 dummy; + void __user *argp = (void __user *)cmd->data; + + switch (cmd->cmd) { + case KVM_PV_ENABLE: { + r = -EINVAL; + if (kvm_s390_pv_is_protected(kvm)) + break; + + /* + * FMT 4 SIE needs esca. As we never switch back to bsca from + * esca, we need no cleanup in the error cases below + */ + r = sca_switch_to_extended(kvm); + if (r) + break; + + down_write(¤t->mm->mmap_sem); + r = gmap_mark_unmergeable(); + up_write(¤t->mm->mmap_sem); + if (r) + break; + + r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc); + if (r) + break; + + r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc); + if (r) + kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy); + + /* we need to block service interrupts from now on */ + set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs); + break; + } + case KVM_PV_DISABLE: { + r = -EINVAL; + if (!kvm_s390_pv_is_protected(kvm)) + break; + + r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc); + /* + * If a CPU could not be destroyed, destroy VM will also fail. + * There is no point in trying to destroy it. Instead return + * the rc and rrc from the first CPU that failed destroying. + */ + if (r) + break; + r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc); + + /* no need to block service interrupts any more */ + clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs); + break; + } + case KVM_PV_SET_SEC_PARMS: { + struct kvm_s390_pv_sec_parm parms = {}; + void *hdr; + + r = -EINVAL; + if (!kvm_s390_pv_is_protected(kvm)) + break; + + r = -EFAULT; + if (copy_from_user(&parms, argp, sizeof(parms))) + break; + + /* Currently restricted to 8KB */ + r = -EINVAL; + if (parms.length > PAGE_SIZE * 2) + break; + + r = -ENOMEM; + hdr = vmalloc(parms.length); + if (!hdr) + break; + + r = -EFAULT; + if (!copy_from_user(hdr, (void __user *)parms.origin, + parms.length)) + r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length, + &cmd->rc, &cmd->rrc); + + vfree(hdr); + break; + } + case KVM_PV_UNPACK: { + struct kvm_s390_pv_unp unp = {}; + + r = -EINVAL; + if (!kvm_s390_pv_is_protected(kvm)) + break; + + r = -EFAULT; + if (copy_from_user(&unp, argp, sizeof(unp))) + break; + + r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak, + &cmd->rc, &cmd->rrc); + break; + } + case KVM_PV_VERIFY: { + r = -EINVAL; + if (!kvm_s390_pv_is_protected(kvm)) + break; + + r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), + UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc); + KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc, + cmd->rrc); + break; + } + case KVM_PV_PREP_RESET: { + r = -EINVAL; + if (!kvm_s390_pv_is_protected(kvm)) + break; + + r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), + UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc); + KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x", + cmd->rc, cmd->rrc); + break; + } + case KVM_PV_UNSHARE_ALL: { + r = -EINVAL; + if (!kvm_s390_pv_is_protected(kvm)) + break; + + r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), + UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc); + KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x", + cmd->rc, cmd->rrc); + break; + } + default: + r = -ENOTTY; + } + return r; +} + long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -2255,6 +2455,33 @@ long kvm_arch_vm_ioctl(struct file *filp, mutex_unlock(&kvm->slots_lock); break; } + case KVM_S390_PV_COMMAND: { + struct kvm_pv_cmd args; + + /* protvirt means user sigp */ + kvm->arch.user_cpu_state_ctrl = 1; + r = 0; + if (!is_prot_virt_host()) { + r = -EINVAL; + break; + } + if (copy_from_user(&args, argp, sizeof(args))) { + r = -EFAULT; + break; + } + if (args.flags) { + r = -EINVAL; + break; + } + mutex_lock(&kvm->lock); + r = kvm_s390_handle_pv(kvm, &args); + mutex_unlock(&kvm->lock); + if (copy_to_user(argp, &args, sizeof(args))) { + r = -EFAULT; + break; + } + break; + } default: r = -ENOTTY; } @@ -2504,7 +2731,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.use_skf = sclp.has_skey; spin_lock_init(&kvm->arch.start_stop_lock); kvm_s390_vsie_init(kvm); - kvm_s390_gisa_init(kvm); + if (use_gisa) + kvm_s390_gisa_init(kvm); KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid); return 0; @@ -2518,6 +2746,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) { + u16 rc, rrc; + VCPU_EVENT(vcpu, 3, "%s", "free cpu"); trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); kvm_s390_clear_local_irqs(vcpu); @@ -2530,6 +2760,9 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) if (vcpu->kvm->arch.use_cmma) kvm_s390_vcpu_unsetup_cmma(vcpu); + /* We can not hold the vcpu mutex here, we are already dying */ + if (kvm_s390_pv_cpu_get_handle(vcpu)) + kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc); free_page((unsigned long)(vcpu->arch.sie_block)); } @@ -2551,10 +2784,20 @@ static void kvm_free_vcpus(struct kvm *kvm) void kvm_arch_destroy_vm(struct kvm *kvm) { + u16 rc, rrc; + kvm_free_vcpus(kvm); sca_dispose(kvm); - debug_unregister(kvm->arch.dbf); kvm_s390_gisa_destroy(kvm); + /* + * We are already at the end of life and kvm->lock is not taken. + * This is ok as the file descriptor is closed by now and nobody + * can mess with the pv state. To avoid lockdep_assert_held from + * complaining we do not use kvm_s390_pv_is_protected. + */ + if (kvm_s390_pv_get_handle(kvm)) + kvm_s390_pv_deinit_vm(kvm, &rc, &rrc); + debug_unregister(kvm->arch.dbf); free_page((unsigned long)kvm->arch.sie_page2); if (!kvm_is_ucontrol(kvm)) gmap_remove(kvm->arch.gmap); @@ -2650,6 +2893,9 @@ static int sca_switch_to_extended(struct kvm *kvm) unsigned int vcpu_idx; u32 scaol, scaoh; + if (kvm->arch.use_esca) + return 0; + new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO); if (!new_sca) return -ENOMEM; @@ -2901,6 +3147,7 @@ static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu) static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu) { int rc = 0; + u16 uvrc, uvrrc; atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | CPUSTAT_SM | @@ -2968,6 +3215,14 @@ static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu) kvm_s390_vcpu_crypto_setup(vcpu); + mutex_lock(&vcpu->kvm->lock); + if (kvm_s390_pv_is_protected(vcpu->kvm)) { + rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc); + if (rc) + kvm_s390_vcpu_unsetup_cmma(vcpu); + } + mutex_unlock(&vcpu->kvm->lock); + return rc; } @@ -3277,7 +3532,6 @@ static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) kvm_s390_set_prefix(vcpu, 0); kvm_s390_set_cpu_timer(vcpu, 0); vcpu->arch.sie_block->ckc = 0; - vcpu->arch.sie_block->todpr = 0; memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr)); vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK; vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK; @@ -3295,9 +3549,17 @@ static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) vcpu->run->s.regs.pp = 0; vcpu->run->s.regs.gbea = 1; vcpu->run->s.regs.fpc = 0; - vcpu->arch.sie_block->gbea = 1; - vcpu->arch.sie_block->pp = 0; - vcpu->arch.sie_block->fpf &= ~FPF_BPBC; + /* + * Do not reset these registers in the protected case, as some of + * them are overlayed and they are not accessible in this case + * anyway. + */ + if (!kvm_s390_pv_cpu_is_protected(vcpu)) { + vcpu->arch.sie_block->gbea = 1; + vcpu->arch.sie_block->pp = 0; + vcpu->arch.sie_block->fpf &= ~FPF_BPBC; + vcpu->arch.sie_block->todpr = 0; + } } static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu) @@ -3487,14 +3749,20 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, switch (mp_state->mp_state) { case KVM_MP_STATE_STOPPED: - kvm_s390_vcpu_stop(vcpu); + rc = kvm_s390_vcpu_stop(vcpu); break; case KVM_MP_STATE_OPERATING: - kvm_s390_vcpu_start(vcpu); + rc = kvm_s390_vcpu_start(vcpu); break; case KVM_MP_STATE_LOAD: + if (!kvm_s390_pv_cpu_is_protected(vcpu)) { + rc = -ENXIO; + break; + } + rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD); + break; case KVM_MP_STATE_CHECK_STOP: - /* fall through - CHECK_STOP and LOAD are not supported yet */ + fallthrough; /* CHECK_STOP and LOAD are not supported yet */ default: rc = -ENXIO; } @@ -3844,9 +4112,11 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) return vcpu_post_run_fault_in_sie(vcpu); } +#define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK) static int __vcpu_run(struct kvm_vcpu *vcpu) { int rc, exit_reason; + struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block; /* * We try to hold kvm->srcu during most of vcpu_run (except when run- @@ -3868,8 +4138,28 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) guest_enter_irqoff(); __disable_cpu_timer_accounting(vcpu); local_irq_enable(); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + memcpy(sie_page->pv_grregs, + vcpu->run->s.regs.gprs, + sizeof(sie_page->pv_grregs)); + } exit_reason = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + memcpy(vcpu->run->s.regs.gprs, + sie_page->pv_grregs, + sizeof(sie_page->pv_grregs)); + /* + * We're not allowed to inject interrupts on intercepts + * that leave the guest state in an "in-between" state + * where the next SIE entry will do a continuation. + * Fence interrupts in our "internal" PSW. + */ + if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR || + vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) { + vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK; + } + } local_irq_disable(); __enable_cpu_timer_accounting(vcpu); guest_exit_irqoff(); @@ -3883,7 +4173,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) return rc; } -static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +static void sync_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { struct runtime_instr_cb *riccb; struct gs_cb *gscb; @@ -3892,16 +4182,7 @@ static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) gscb = (struct gs_cb *) &kvm_run->s.regs.gscb; vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask; vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr; - if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) - kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); - if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { - memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); - /* some control register changes require a tlb flush */ - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); - } if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) { - kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm); - vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc; vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr; vcpu->arch.sie_block->pp = kvm_run->s.regs.pp; vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea; @@ -3942,20 +4223,6 @@ static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu->arch.sie_block->fpf &= ~FPF_BPBC; vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0; } - save_access_regs(vcpu->arch.host_acrs); - restore_access_regs(vcpu->run->s.regs.acrs); - /* save host (userspace) fprs/vrs */ - save_fpu_regs(); - vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc; - vcpu->arch.host_fpregs.regs = current->thread.fpu.regs; - if (MACHINE_HAS_VX) - current->thread.fpu.regs = vcpu->run->s.regs.vrs; - else - current->thread.fpu.regs = vcpu->run->s.regs.fprs; - current->thread.fpu.fpc = vcpu->run->s.regs.fpc; - if (test_fp_ctl(current->thread.fpu.fpc)) - /* User space provided an invalid FPC, let's clear it */ - current->thread.fpu.fpc = 0; if (MACHINE_HAS_GS) { preempt_disable(); __ctl_set_bit(2, 4); @@ -3971,33 +4238,63 @@ static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) preempt_enable(); } /* SIE will load etoken directly from SDNX and therefore kvm_run */ +} + +static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) + kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); + if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { + memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); + /* some control register changes require a tlb flush */ + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + } + if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) { + kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm); + vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc; + } + save_access_regs(vcpu->arch.host_acrs); + restore_access_regs(vcpu->run->s.regs.acrs); + /* save host (userspace) fprs/vrs */ + save_fpu_regs(); + vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc; + vcpu->arch.host_fpregs.regs = current->thread.fpu.regs; + if (MACHINE_HAS_VX) + current->thread.fpu.regs = vcpu->run->s.regs.vrs; + else + current->thread.fpu.regs = vcpu->run->s.regs.fprs; + current->thread.fpu.fpc = vcpu->run->s.regs.fpc; + if (test_fp_ctl(current->thread.fpu.fpc)) + /* User space provided an invalid FPC, let's clear it */ + current->thread.fpu.fpc = 0; + + /* Sync fmt2 only data */ + if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) { + sync_regs_fmt2(vcpu, kvm_run); + } else { + /* + * In several places we have to modify our internal view to + * not do things that are disallowed by the ultravisor. For + * example we must not inject interrupts after specific exits + * (e.g. 112 prefix page not secure). We do this by turning + * off the machine check, external and I/O interrupt bits + * of our PSW copy. To avoid getting validity intercepts, we + * do only accept the condition code from userspace. + */ + vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC; + vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask & + PSW_MASK_CC; + } kvm_run->kvm_dirty_regs = 0; } -static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +static void store_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { - kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; - kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; - kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu); - memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); - kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu); - kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc; kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr; kvm_run->s.regs.pp = vcpu->arch.sie_block->pp; kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea; - kvm_run->s.regs.pft = vcpu->arch.pfault_token; - kvm_run->s.regs.pfs = vcpu->arch.pfault_select; - kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC; - save_access_regs(vcpu->run->s.regs.acrs); - restore_access_regs(vcpu->arch.host_acrs); - /* Save guest register state */ - save_fpu_regs(); - vcpu->run->s.regs.fpc = current->thread.fpu.fpc; - /* Restore will be done lazily at return */ - current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc; - current->thread.fpu.regs = vcpu->arch.host_fpregs.regs; if (MACHINE_HAS_GS) { __ctl_set_bit(2, 4); if (vcpu->arch.gs_enabled) @@ -4013,6 +4310,29 @@ static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) /* SIE will save etoken directly into SDNX and therefore kvm_run */ } +static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; + kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; + kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu); + memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); + kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu); + kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc; + kvm_run->s.regs.pft = vcpu->arch.pfault_token; + kvm_run->s.regs.pfs = vcpu->arch.pfault_select; + kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; + save_access_regs(vcpu->run->s.regs.acrs); + restore_access_regs(vcpu->arch.host_acrs); + /* Save guest register state */ + save_fpu_regs(); + vcpu->run->s.regs.fpc = current->thread.fpu.fpc; + /* Restore will be done lazily at return */ + current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc; + current->thread.fpu.regs = vcpu->arch.host_fpregs.regs; + if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) + store_regs_fmt2(vcpu, kvm_run); +} + int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { int rc; @@ -4034,6 +4354,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) kvm_sigset_activate(vcpu); + /* + * no need to check the return value of vcpu_start as it can only have + * an error for protvirt, but protvirt means user cpu state + */ if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) { kvm_s390_vcpu_start(vcpu); } else if (is_vcpu_stopped(vcpu)) { @@ -4171,18 +4495,27 @@ static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu) kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu); } -void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) +int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) { - int i, online_vcpus, started_vcpus = 0; + int i, online_vcpus, r = 0, started_vcpus = 0; if (!is_vcpu_stopped(vcpu)) - return; + return 0; trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1); /* Only one cpu at a time may enter/leave the STOPPED state. */ spin_lock(&vcpu->kvm->arch.start_stop_lock); online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); + /* Let's tell the UV that we want to change into the operating state */ + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR); + if (r) { + spin_unlock(&vcpu->kvm->arch.start_stop_lock); + return r; + } + } + for (i = 0; i < online_vcpus; i++) { if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) started_vcpus++; @@ -4201,28 +4534,44 @@ void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) } kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED); + /* + * The real PSW might have changed due to a RESTART interpreted by the + * ultravisor. We block all interrupts and let the next sie exit + * refresh our view. + */ + if (kvm_s390_pv_cpu_is_protected(vcpu)) + vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK; /* * Another VCPU might have used IBS while we were offline. * Let's play safe and flush the VCPU at startup. */ kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); spin_unlock(&vcpu->kvm->arch.start_stop_lock); - return; + return 0; } -void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) +int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) { - int i, online_vcpus, started_vcpus = 0; + int i, online_vcpus, r = 0, started_vcpus = 0; struct kvm_vcpu *started_vcpu = NULL; if (is_vcpu_stopped(vcpu)) - return; + return 0; trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0); /* Only one cpu at a time may enter/leave the STOPPED state. */ spin_lock(&vcpu->kvm->arch.start_stop_lock); online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); + /* Let's tell the UV that we want to change into the stopped state */ + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP); + if (r) { + spin_unlock(&vcpu->kvm->arch.start_stop_lock); + return r; + } + } + /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ kvm_s390_clear_stop_irq(vcpu); @@ -4245,7 +4594,7 @@ void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) } spin_unlock(&vcpu->kvm->arch.start_stop_lock); - return; + return 0; } static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, @@ -4272,12 +4621,40 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, return r; } +static long kvm_s390_guest_sida_op(struct kvm_vcpu *vcpu, + struct kvm_s390_mem_op *mop) +{ + void __user *uaddr = (void __user *)mop->buf; + int r = 0; + + if (mop->flags || !mop->size) + return -EINVAL; + if (mop->size + mop->sida_offset < mop->size) + return -EINVAL; + if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block)) + return -E2BIG; + + switch (mop->op) { + case KVM_S390_MEMOP_SIDA_READ: + if (copy_to_user(uaddr, (void *)(sida_origin(vcpu->arch.sie_block) + + mop->sida_offset), mop->size)) + r = -EFAULT; + + break; + case KVM_S390_MEMOP_SIDA_WRITE: + if (copy_from_user((void *)(sida_origin(vcpu->arch.sie_block) + + mop->sida_offset), uaddr, mop->size)) + r = -EFAULT; + break; + } + return r; +} static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu, struct kvm_s390_mem_op *mop) { void __user *uaddr = (void __user *)mop->buf; void *tmpbuf = NULL; - int r, srcu_idx; + int r = 0; const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION | KVM_S390_MEMOP_F_CHECK_ONLY; @@ -4287,14 +4664,15 @@ static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu, if (mop->size > MEM_OP_MAX_SIZE) return -E2BIG; + if (kvm_s390_pv_cpu_is_protected(vcpu)) + return -EINVAL; + if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) { tmpbuf = vmalloc(mop->size); if (!tmpbuf) return -ENOMEM; } - srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); - switch (mop->op) { case KVM_S390_MEMOP_LOGICAL_READ: if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) { @@ -4320,12 +4698,8 @@ static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu, } r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size); break; - default: - r = -EINVAL; } - srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); - if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0) kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); @@ -4333,6 +4707,31 @@ static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu, return r; } +static long kvm_s390_guest_memsida_op(struct kvm_vcpu *vcpu, + struct kvm_s390_mem_op *mop) +{ + int r, srcu_idx; + + srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); + + switch (mop->op) { + case KVM_S390_MEMOP_LOGICAL_READ: + case KVM_S390_MEMOP_LOGICAL_WRITE: + r = kvm_s390_guest_mem_op(vcpu, mop); + break; + case KVM_S390_MEMOP_SIDA_READ: + case KVM_S390_MEMOP_SIDA_WRITE: + /* we are locked against sida going away by the vcpu->mutex */ + r = kvm_s390_guest_sida_op(vcpu, mop); + break; + default: + r = -EINVAL; + } + + srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); + return r; +} + long kvm_arch_vcpu_async_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -4368,6 +4767,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, void __user *argp = (void __user *)arg; int idx; long r; + u16 rc, rrc; vcpu_load(vcpu); @@ -4389,18 +4789,40 @@ long kvm_arch_vcpu_ioctl(struct file *filp, case KVM_S390_CLEAR_RESET: r = 0; kvm_arch_vcpu_ioctl_clear_reset(vcpu); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), + UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc); + VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x", + rc, rrc); + } break; case KVM_S390_INITIAL_RESET: r = 0; kvm_arch_vcpu_ioctl_initial_reset(vcpu); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), + UVC_CMD_CPU_RESET_INITIAL, + &rc, &rrc); + VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x", + rc, rrc); + } break; case KVM_S390_NORMAL_RESET: r = 0; kvm_arch_vcpu_ioctl_normal_reset(vcpu); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), + UVC_CMD_CPU_RESET, &rc, &rrc); + VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x", + rc, rrc); + } break; case KVM_SET_ONE_REG: case KVM_GET_ONE_REG: { struct kvm_one_reg reg; + r = -EINVAL; + if (kvm_s390_pv_cpu_is_protected(vcpu)) + break; r = -EFAULT; if (copy_from_user(®, argp, sizeof(reg))) break; @@ -4463,7 +4885,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, struct kvm_s390_mem_op mem_op; if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0) - r = kvm_s390_guest_mem_op(vcpu, &mem_op); + r = kvm_s390_guest_memsida_op(vcpu, &mem_op); else r = -EFAULT; break; @@ -4523,12 +4945,6 @@ vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } -int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned long npages) -{ - return 0; -} - /* Section: memory related */ int kvm_arch_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *memslot, @@ -4549,12 +4965,15 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit) return -EINVAL; + /* When we are protected, we should not change the memory slots */ + if (kvm_s390_pv_get_handle(kvm)) + return -EINVAL; return 0; } void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { @@ -4570,7 +4989,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, old->npages * PAGE_SIZE); if (rc) break; - /* FALLTHROUGH */ + fallthrough; case KVM_MR_CREATE: rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, mem->guest_phys_addr, mem->memory_size); diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index 6d9448dbd052..79dcd647b378 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -2,7 +2,7 @@ /* * definition for kvm on s390 * - * Copyright IBM Corp. 2008, 2009 + * Copyright IBM Corp. 2008, 2020 * * Author(s): Carsten Otte * Christian Borntraeger @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,17 @@ #define IS_ITDB_VALID(vcpu) ((*(char *)vcpu->arch.sie_block->itdba == TDB_FORMAT1)) extern debug_info_t *kvm_s390_dbf; +extern debug_info_t *kvm_s390_dbf_uv; + +#define KVM_UV_EVENT(d_kvm, d_loglevel, d_string, d_args...)\ +do { \ + debug_sprintf_event((d_kvm)->arch.dbf, d_loglevel, d_string "\n", \ + d_args); \ + debug_sprintf_event(kvm_s390_dbf_uv, d_loglevel, \ + "%d: " d_string "\n", (d_kvm)->userspace_pid, \ + d_args); \ +} while (0) + #define KVM_EVENT(d_loglevel, d_string, d_args...)\ do { \ debug_sprintf_event(kvm_s390_dbf, d_loglevel, d_string "\n", \ @@ -196,6 +208,39 @@ static inline int kvm_s390_user_cpu_state_ctrl(struct kvm *kvm) return kvm->arch.user_cpu_state_ctrl != 0; } +/* implemented in pv.c */ +int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc); +int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc); +int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc); +int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc); +int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length, u16 *rc, + u16 *rrc); +int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size, + unsigned long tweak, u16 *rc, u16 *rrc); +int kvm_s390_pv_set_cpu_state(struct kvm_vcpu *vcpu, u8 state); + +static inline u64 kvm_s390_pv_get_handle(struct kvm *kvm) +{ + return kvm->arch.pv.handle; +} + +static inline u64 kvm_s390_pv_cpu_get_handle(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.pv.handle; +} + +static inline bool kvm_s390_pv_is_protected(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->lock); + return !!kvm_s390_pv_get_handle(kvm); +} + +static inline bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu) +{ + lockdep_assert_held(&vcpu->mutex); + return !!kvm_s390_pv_cpu_get_handle(vcpu); +} + /* implemented in interrupt.c */ int kvm_s390_handle_wait(struct kvm_vcpu *vcpu); void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu); @@ -286,8 +331,8 @@ void kvm_s390_set_tod_clock(struct kvm *kvm, long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable); int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr); int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr); -void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu); -void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu); +int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu); +int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu); void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu); void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu); bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu); diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index ed52ffa8d5d4..69a824f9ef0b 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -2,7 +2,7 @@ /* * handling privileged instructions * - * Copyright IBM Corp. 2008, 2018 + * Copyright IBM Corp. 2008, 2020 * * Author(s): Carsten Otte * Christian Borntraeger @@ -872,7 +872,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu) operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); - if (operand2 & 0xfff) + if (!kvm_s390_pv_cpu_is_protected(vcpu) && (operand2 & 0xfff)) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); switch (fc) { @@ -893,8 +893,13 @@ static int handle_stsi(struct kvm_vcpu *vcpu) handle_stsi_3_2_2(vcpu, (void *) mem); break; } - - rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE); + if (kvm_s390_pv_cpu_is_protected(vcpu)) { + memcpy((void *)sida_origin(vcpu->arch.sie_block), (void *)mem, + PAGE_SIZE); + rc = 0; + } else { + rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE); + } if (rc) { rc = kvm_s390_inject_prog_cond(vcpu, rc); goto out; diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c new file mode 100644 index 000000000000..63e330109b63 --- /dev/null +++ b/arch/s390/kvm/pv.c @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hosting Protected Virtual Machines + * + * Copyright IBM Corp. 2019, 2020 + * Author(s): Janosch Frank + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvm-s390.h" + +int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) +{ + int cc = 0; + + if (kvm_s390_pv_cpu_get_handle(vcpu)) { + cc = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), + UVC_CMD_DESTROY_SEC_CPU, rc, rrc); + + KVM_UV_EVENT(vcpu->kvm, 3, + "PROTVIRT DESTROY VCPU %d: rc %x rrc %x", + vcpu->vcpu_id, *rc, *rrc); + WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", + *rc, *rrc); + } + /* Intended memory leak for something that should never happen. */ + if (!cc) + free_pages(vcpu->arch.pv.stor_base, + get_order(uv_info.guest_cpu_stor_len)); + + free_page(sida_origin(vcpu->arch.sie_block)); + vcpu->arch.sie_block->pv_handle_cpu = 0; + vcpu->arch.sie_block->pv_handle_config = 0; + memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); + vcpu->arch.sie_block->sdf = 0; + /* + * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). + * Use the reset value of gbea to avoid leaking the kernel pointer of + * the just freed sida. + */ + vcpu->arch.sie_block->gbea = 1; + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + + return cc ? EIO : 0; +} + +int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) +{ + struct uv_cb_csc uvcb = { + .header.cmd = UVC_CMD_CREATE_SEC_CPU, + .header.len = sizeof(uvcb), + }; + int cc; + + if (kvm_s390_pv_cpu_get_handle(vcpu)) + return -EINVAL; + + vcpu->arch.pv.stor_base = __get_free_pages(GFP_KERNEL, + get_order(uv_info.guest_cpu_stor_len)); + if (!vcpu->arch.pv.stor_base) + return -ENOMEM; + + /* Input */ + uvcb.guest_handle = kvm_s390_pv_get_handle(vcpu->kvm); + uvcb.num = vcpu->arch.sie_block->icpua; + uvcb.state_origin = (u64)vcpu->arch.sie_block; + uvcb.stor_origin = (u64)vcpu->arch.pv.stor_base; + + /* Alloc Secure Instruction Data Area Designation */ + vcpu->arch.sie_block->sidad = __get_free_page(GFP_KERNEL | __GFP_ZERO); + if (!vcpu->arch.sie_block->sidad) { + free_pages(vcpu->arch.pv.stor_base, + get_order(uv_info.guest_cpu_stor_len)); + return -ENOMEM; + } + + cc = uv_call(0, (u64)&uvcb); + *rc = uvcb.header.rc; + *rrc = uvcb.header.rrc; + KVM_UV_EVENT(vcpu->kvm, 3, + "PROTVIRT CREATE VCPU: cpu %d handle %llx rc %x rrc %x", + vcpu->vcpu_id, uvcb.cpu_handle, uvcb.header.rc, + uvcb.header.rrc); + + if (cc) { + u16 dummy; + + kvm_s390_pv_destroy_cpu(vcpu, &dummy, &dummy); + return -EIO; + } + + /* Output */ + vcpu->arch.pv.handle = uvcb.cpu_handle; + vcpu->arch.sie_block->pv_handle_cpu = uvcb.cpu_handle; + vcpu->arch.sie_block->pv_handle_config = kvm_s390_pv_get_handle(vcpu->kvm); + vcpu->arch.sie_block->sdf = 2; + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + return 0; +} + +/* only free resources when the destroy was successful */ +static void kvm_s390_pv_dealloc_vm(struct kvm *kvm) +{ + vfree(kvm->arch.pv.stor_var); + free_pages(kvm->arch.pv.stor_base, + get_order(uv_info.guest_base_stor_len)); + memset(&kvm->arch.pv, 0, sizeof(kvm->arch.pv)); +} + +static int kvm_s390_pv_alloc_vm(struct kvm *kvm) +{ + unsigned long base = uv_info.guest_base_stor_len; + unsigned long virt = uv_info.guest_virt_var_stor_len; + unsigned long npages = 0, vlen = 0; + struct kvm_memory_slot *memslot; + + kvm->arch.pv.stor_var = NULL; + kvm->arch.pv.stor_base = __get_free_pages(GFP_KERNEL, get_order(base)); + if (!kvm->arch.pv.stor_base) + return -ENOMEM; + + /* + * Calculate current guest storage for allocation of the + * variable storage, which is based on the length in MB. + * + * Slots are sorted by GFN + */ + mutex_lock(&kvm->slots_lock); + memslot = kvm_memslots(kvm)->memslots; + npages = memslot->base_gfn + memslot->npages; + mutex_unlock(&kvm->slots_lock); + + kvm->arch.pv.guest_len = npages * PAGE_SIZE; + + /* Allocate variable storage */ + vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE); + vlen += uv_info.guest_virt_base_stor_len; + kvm->arch.pv.stor_var = vzalloc(vlen); + if (!kvm->arch.pv.stor_var) + goto out_err; + return 0; + +out_err: + kvm_s390_pv_dealloc_vm(kvm); + return -ENOMEM; +} + +/* this should not fail, but if it does, we must not free the donated memory */ +int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc) +{ + int cc; + + /* make all pages accessible before destroying the guest */ + s390_reset_acc(kvm->mm); + + cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), + UVC_CMD_DESTROY_SEC_CONF, rc, rrc); + WRITE_ONCE(kvm->arch.gmap->guest_handle, 0); + atomic_set(&kvm->mm->context.is_protected, 0); + KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM: rc %x rrc %x", *rc, *rrc); + WARN_ONCE(cc, "protvirt destroy vm failed rc %x rrc %x", *rc, *rrc); + /* Inteded memory leak on "impossible" error */ + if (!cc) + kvm_s390_pv_dealloc_vm(kvm); + return cc ? -EIO : 0; +} + +int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) +{ + struct uv_cb_cgc uvcb = { + .header.cmd = UVC_CMD_CREATE_SEC_CONF, + .header.len = sizeof(uvcb) + }; + int cc, ret; + u16 dummy; + + ret = kvm_s390_pv_alloc_vm(kvm); + if (ret) + return ret; + + /* Inputs */ + uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */ + uvcb.guest_stor_len = kvm->arch.pv.guest_len; + uvcb.guest_asce = kvm->arch.gmap->asce; + uvcb.guest_sca = (unsigned long)kvm->arch.sca; + uvcb.conf_base_stor_origin = (u64)kvm->arch.pv.stor_base; + uvcb.conf_virt_stor_origin = (u64)kvm->arch.pv.stor_var; + + cc = uv_call(0, (u64)&uvcb); + *rc = uvcb.header.rc; + *rrc = uvcb.header.rrc; + KVM_UV_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x", + uvcb.guest_handle, uvcb.guest_stor_len, *rc, *rrc); + + /* Outputs */ + kvm->arch.pv.handle = uvcb.guest_handle; + + if (cc) { + if (uvcb.header.rc & UVC_RC_NEED_DESTROY) + kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy); + else + kvm_s390_pv_dealloc_vm(kvm); + return -EIO; + } + kvm->arch.gmap->guest_handle = uvcb.guest_handle; + atomic_set(&kvm->mm->context.is_protected, 1); + return 0; +} + +int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length, u16 *rc, + u16 *rrc) +{ + struct uv_cb_ssc uvcb = { + .header.cmd = UVC_CMD_SET_SEC_CONF_PARAMS, + .header.len = sizeof(uvcb), + .sec_header_origin = (u64)hdr, + .sec_header_len = length, + .guest_handle = kvm_s390_pv_get_handle(kvm), + }; + int cc = uv_call(0, (u64)&uvcb); + + *rc = uvcb.header.rc; + *rrc = uvcb.header.rrc; + KVM_UV_EVENT(kvm, 3, "PROTVIRT VM SET PARMS: rc %x rrc %x", + *rc, *rrc); + return cc ? -EINVAL : 0; +} + +static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak, + u64 offset, u16 *rc, u16 *rrc) +{ + struct uv_cb_unp uvcb = { + .header.cmd = UVC_CMD_UNPACK_IMG, + .header.len = sizeof(uvcb), + .guest_handle = kvm_s390_pv_get_handle(kvm), + .gaddr = addr, + .tweak[0] = tweak, + .tweak[1] = offset, + }; + int ret = gmap_make_secure(kvm->arch.gmap, addr, &uvcb); + + *rc = uvcb.header.rc; + *rrc = uvcb.header.rrc; + + if (ret && ret != -EAGAIN) + KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x", + uvcb.gaddr, *rc, *rrc); + return ret; +} + +int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size, + unsigned long tweak, u16 *rc, u16 *rrc) +{ + u64 offset = 0; + int ret = 0; + + if (addr & ~PAGE_MASK || !size || size & ~PAGE_MASK) + return -EINVAL; + + KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: start addr %lx size %lx", + addr, size); + + while (offset < size) { + ret = unpack_one(kvm, addr, tweak, offset, rc, rrc); + if (ret == -EAGAIN) { + cond_resched(); + if (fatal_signal_pending(current)) + break; + continue; + } + if (ret) + break; + addr += PAGE_SIZE; + offset += PAGE_SIZE; + } + if (!ret) + KVM_UV_EVENT(kvm, 3, "%s", "PROTVIRT VM UNPACK: successful"); + return ret; +} + +int kvm_s390_pv_set_cpu_state(struct kvm_vcpu *vcpu, u8 state) +{ + struct uv_cb_cpu_set_state uvcb = { + .header.cmd = UVC_CMD_CPU_SET_STATE, + .header.len = sizeof(uvcb), + .cpu_handle = kvm_s390_pv_cpu_get_handle(vcpu), + .state = state, + }; + int cc; + + cc = uv_call(0, (u64)&uvcb); + KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT SET CPU %d STATE %d rc %x rrc %x", + vcpu->vcpu_id, state, uvcb.header.rc, uvcb.header.rrc); + if (cc) + return -EINVAL; + return 0; +} diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 7b0bb475c166..09bf7f2121ac 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "../kernel/entry.h" #define __FAIL_ADDR_MASK -4096L @@ -429,7 +430,7 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) address = trans_exc_code & __FAIL_ADDR_MASK; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); - flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + flags = FAULT_FLAG_DEFAULT; if (user_mode(regs)) flags |= FAULT_FLAG_USER; if (access == VM_WRITE || (trans_exc_code & store_indication) == 0x400) @@ -480,8 +481,7 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) * the fault. */ fault = handle_mm_fault(vma, address, flags); - /* No reason to continue if interrupted by SIGKILL. */ - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { + if (fault_signal_pending(fault, regs)) { fault = VM_FAULT_SIGNAL; if (flags & FAULT_FLAG_RETRY_NOWAIT) goto out_up; @@ -514,10 +514,7 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) fault = VM_FAULT_PFAULT; goto out_up; } - /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk - * of starvation. */ - flags &= ~(FAULT_FLAG_ALLOW_RETRY | - FAULT_FLAG_RETRY_NOWAIT); + flags &= ~FAULT_FLAG_RETRY_NOWAIT; flags |= FAULT_FLAG_TRIED; down_read(&mm->mmap_sem); goto retry; @@ -816,3 +813,80 @@ static int __init pfault_irq_init(void) early_initcall(pfault_irq_init); #endif /* CONFIG_PFAULT */ + +#if IS_ENABLED(CONFIG_PGSTE) +void do_secure_storage_access(struct pt_regs *regs) +{ + unsigned long addr = regs->int_parm_long & __FAIL_ADDR_MASK; + struct vm_area_struct *vma; + struct mm_struct *mm; + struct page *page; + int rc; + + switch (get_fault_type(regs)) { + case USER_FAULT: + mm = current->mm; + down_read(&mm->mmap_sem); + vma = find_vma(mm, addr); + if (!vma) { + up_read(&mm->mmap_sem); + do_fault_error(regs, VM_READ | VM_WRITE, VM_FAULT_BADMAP); + break; + } + page = follow_page(vma, addr, FOLL_WRITE | FOLL_GET); + if (IS_ERR_OR_NULL(page)) { + up_read(&mm->mmap_sem); + break; + } + if (arch_make_page_accessible(page)) + send_sig(SIGSEGV, current, 0); + put_page(page); + up_read(&mm->mmap_sem); + break; + case KERNEL_FAULT: + page = phys_to_page(addr); + if (unlikely(!try_get_page(page))) + break; + rc = arch_make_page_accessible(page); + put_page(page); + if (rc) + BUG(); + break; + case VDSO_FAULT: + /* fallthrough */ + case GMAP_FAULT: + /* fallthrough */ + default: + do_fault_error(regs, VM_READ | VM_WRITE, VM_FAULT_BADMAP); + WARN_ON_ONCE(1); + } +} +NOKPROBE_SYMBOL(do_secure_storage_access); + +void do_non_secure_storage_access(struct pt_regs *regs) +{ + unsigned long gaddr = regs->int_parm_long & __FAIL_ADDR_MASK; + struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; + + if (get_fault_type(regs) != GMAP_FAULT) { + do_fault_error(regs, VM_READ | VM_WRITE, VM_FAULT_BADMAP); + WARN_ON_ONCE(1); + return; + } + + if (gmap_convert_to_secure(gmap, gaddr) == -EINVAL) + send_sig(SIGSEGV, current, 0); +} +NOKPROBE_SYMBOL(do_non_secure_storage_access); + +#else +void do_secure_storage_access(struct pt_regs *regs) +{ + default_trap_handler(regs); +} + +void do_non_secure_storage_access(struct pt_regs *regs) +{ + default_trap_handler(regs); +} +#endif diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index edcdca97e85e..2fbece47ef6f 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -804,7 +804,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap, if (*table & _REGION_ENTRY_INVALID) return NULL; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); - /* Fallthrough */ + fallthrough; case _ASCE_TYPE_REGION2: table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT; if (level == 3) @@ -812,7 +812,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap, if (*table & _REGION_ENTRY_INVALID) return NULL; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); - /* Fallthrough */ + fallthrough; case _ASCE_TYPE_REGION3: table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT; if (level == 2) @@ -820,7 +820,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap, if (*table & _REGION_ENTRY_INVALID) return NULL; table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); - /* Fallthrough */ + fallthrough; case _ASCE_TYPE_SEGMENT: table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT; if (level == 1) @@ -2548,6 +2548,23 @@ int s390_enable_sie(void) } EXPORT_SYMBOL_GPL(s390_enable_sie); +int gmap_mark_unmergeable(void) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + int ret; + + for (vma = mm->mmap; vma; vma = vma->vm_next) { + ret = ksm_madvise(vma, vma->vm_start, vma->vm_end, + MADV_UNMERGEABLE, &vma->vm_flags); + if (ret) + return ret; + } + mm->def_flags &= ~VM_MERGEABLE; + return 0; +} +EXPORT_SYMBOL_GPL(gmap_mark_unmergeable); + /* * Enable storage key handling from now on and initialize the storage * keys with the default key. @@ -2593,7 +2610,6 @@ static const struct mm_walk_ops enable_skey_walk_ops = { int s390_enable_skey(void) { struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; int rc = 0; down_write(&mm->mmap_sem); @@ -2601,16 +2617,11 @@ int s390_enable_skey(void) goto out_up; mm->context.uses_skeys = 1; - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (ksm_madvise(vma, vma->vm_start, vma->vm_end, - MADV_UNMERGEABLE, &vma->vm_flags)) { - mm->context.uses_skeys = 0; - rc = -ENOMEM; - goto out_up; - } + rc = gmap_mark_unmergeable(); + if (rc) { + mm->context.uses_skeys = 0; + goto out_up; } - mm->def_flags &= ~VM_MERGEABLE; - walk_page_range(mm, 0, TASK_SIZE, &enable_skey_walk_ops, NULL); out_up: @@ -2640,3 +2651,38 @@ void s390_reset_cmma(struct mm_struct *mm) up_write(&mm->mmap_sem); } EXPORT_SYMBOL_GPL(s390_reset_cmma); + +/* + * make inaccessible pages accessible again + */ +static int __s390_reset_acc(pte_t *ptep, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t pte = READ_ONCE(*ptep); + + if (pte_present(pte)) + WARN_ON_ONCE(uv_convert_from_secure(pte_val(pte) & PAGE_MASK)); + return 0; +} + +static const struct mm_walk_ops reset_acc_walk_ops = { + .pte_entry = __s390_reset_acc, +}; + +#include +void s390_reset_acc(struct mm_struct *mm) +{ + /* + * we might be called during + * reset: we walk the pages and clear + * close of all kvm file descriptors: we walk the pages and clear + * exit of process on fd closure: vma already gone, do nothing + */ + if (!mmget_not_zero(mm)) + return; + down_read(&mm->mmap_sem); + walk_page_range(mm, 0, TASK_SIZE, &reset_acc_walk_ops, NULL); + up_read(&mm->mmap_sem); + mmput(mm); +} +EXPORT_SYMBOL_GPL(s390_reset_acc); diff --git a/arch/sh/include/asm/Kbuild b/arch/sh/include/asm/Kbuild index 51a54df22c11..7435182ef846 100644 --- a/arch/sh/include/asm/Kbuild +++ b/arch/sh/include/asm/Kbuild @@ -1,22 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += syscall_table.h -generic-y += compat.h -generic-y += current.h -generic-y += delay.h -generic-y += div64.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h -generic-y += irq_regs.h -generic-y += irq_work.h generic-y += kvm_para.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h generic-y += parport.h -generic-y += percpu.h -generic-y += preempt.h -generic-y += serial.h -generic-y += trace_clock.h -generic-y += xor.h diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 5f51456f4fc7..13ee4d20e622 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -302,25 +302,25 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code, * Pagefault was interrupted by SIGKILL. We have no reason to * continue pagefault. */ - if (fatal_signal_pending(current)) { - if (!(fault & VM_FAULT_RETRY)) - up_read(¤t->mm->mmap_sem); + if (fault_signal_pending(fault, regs)) { if (!user_mode(regs)) no_context(regs, error_code, address); return 1; } + /* Release mmap_sem first if necessary */ + if (!(fault & VM_FAULT_RETRY)) + up_read(¤t->mm->mmap_sem); + if (!(fault & VM_FAULT_ERROR)) return 0; if (fault & VM_FAULT_OOM) { /* Kernel mode? Handle exceptions or die: */ if (!user_mode(regs)) { - up_read(¤t->mm->mmap_sem); no_context(regs, error_code, address); return 1; } - up_read(¤t->mm->mmap_sem); /* * We ran out of memory, call the OOM killer, and return the @@ -380,7 +380,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, struct mm_struct *mm; struct vm_area_struct * vma; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; tsk = current; mm = tsk->mm; @@ -481,7 +481,6 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, regs, address); } if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild index 62de2eb2773d..5269a704801f 100644 --- a/arch/sparc/include/asm/Kbuild +++ b/arch/sparc/include/asm/Kbuild @@ -4,21 +4,7 @@ generated-y += syscall_table_32.h generated-y += syscall_table_64.h generated-y += syscall_table_c32.h -generic-y += div64.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += export.h -generic-y += irq_regs.h -generic-y += irq_work.h generic-y += kvm_para.h -generic-y += linkage.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += module.h -generic-y += preempt.h -generic-y += serial.h -generic-y += trace_clock.h -generic-y += word-at-a-time.h diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index 89976c9b936c..f6e0e601f857 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -168,7 +168,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, int from_user = !(regs->psr & PSR_PS); int code; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; if (text_fault) address = regs->pc; @@ -237,7 +237,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, */ fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -261,7 +261,6 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, 1, regs, address); } if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* No need to up_read(&mm->mmap_sem) as we would diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 8b7ddbd14b65..c0c0dd471b6b 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -271,7 +271,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) int si_code, fault_code; vm_fault_t fault; unsigned long address, mm_rss; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; fault_code = get_thread_fault_code(); @@ -425,7 +425,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) goto exit_exception; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -449,7 +449,6 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) 1, regs, address); } if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* No need to up_read(&mm->mmap_sem) as we would diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index 818553064f04..8f18cf56b3dd 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -33,7 +33,7 @@ int handle_page_fault(unsigned long address, unsigned long ip, pmd_t *pmd; pte_t *pte; int err = -EFAULT; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; *code_out = SEGV_MAPERR; @@ -97,7 +97,6 @@ int handle_page_fault(unsigned long address, unsigned long ip, else current->min_flt++; if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; goto retry; diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild index 98aa125a8f06..55026e8240d8 100644 --- a/arch/unicore32/include/asm/Kbuild +++ b/arch/unicore32/include/asm/Kbuild @@ -1,41 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -generic-y += atomic.h -generic-y += bugs.h -generic-y += compat.h -generic-y += current.h -generic-y += device.h -generic-y += div64.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += extable.h -generic-y += fb.h -generic-y += ftrace.h -generic-y += futex.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += local.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h -generic-y += module.h generic-y += parport.h -generic-y += percpu.h -generic-y += preempt.h -generic-y += sections.h -generic-y += serial.h -generic-y += shmparam.h generic-y += syscalls.h -generic-y += topology.h -generic-y += trace_clock.h -generic-y += unaligned.h generic-y += user.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/unicore32/mm/fault.c b/arch/unicore32/mm/fault.c index 76342de9cf8c..a9bd08fbe588 100644 --- a/arch/unicore32/mm/fault.c +++ b/arch/unicore32/mm/fault.c @@ -202,7 +202,7 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs) struct mm_struct *mm; int sig, code; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; tsk = current; mm = tsk->mm; @@ -250,7 +250,7 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * signal first. We do not need to release the mmap_sem because * it would already be released in __lock_page_or_retry in * mm/filemap.c. */ - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return 0; if (!(fault & VM_FAULT_ERROR) && (flags & FAULT_FLAG_ALLOW_RETRY)) { @@ -259,9 +259,7 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs) else tsk->min_flt++; if (fault & VM_FAULT_RETRY) { - /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk - * of starvation. */ - flags &= ~FAULT_FLAG_ALLOW_RETRY; + flags |= FAULT_FLAG_TRIED; goto retry; } } diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 439fcd68c795..5b9d2f47b375 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -232,6 +232,7 @@ config X86 select VIRT_TO_BUS select X86_FEATURE_NAMES if PROC_FS select PROC_PID_ARCH_STATUS if PROC_FS + imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI config INSTRUCTION_DECODER def_bool y diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild index ea34464d6221..b19ec8282d50 100644 --- a/arch/x86/include/asm/Kbuild +++ b/arch/x86/include/asm/Kbuild @@ -10,5 +10,3 @@ generated-y += xen-hypercalls.h generic-y += early_ioremap.h generic-y += export.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index 062cdecb2f24..53f246e9df5a 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -54,7 +54,7 @@ arch_set_bit(long nr, volatile unsigned long *addr) if (__builtin_constant_p(nr)) { asm volatile(LOCK_PREFIX "orb %1,%0" : CONST_MASK_ADDR(nr, addr) - : "iq" ((u8)CONST_MASK(nr)) + : "iq" (CONST_MASK(nr) & 0xff) : "memory"); } else { asm volatile(LOCK_PREFIX __ASM_SIZE(bts) " %1,%0" @@ -74,7 +74,7 @@ arch_clear_bit(long nr, volatile unsigned long *addr) if (__builtin_constant_p(nr)) { asm volatile(LOCK_PREFIX "andb %1,%0" : CONST_MASK_ADDR(nr, addr) - : "iq" ((u8)~CONST_MASK(nr))); + : "iq" (CONST_MASK(nr) ^ 0xff)); } else { asm volatile(LOCK_PREFIX __ASM_SIZE(btr) " %1,%0" : : RLONG_ADDR(addr), "Ir" (nr) : "memory"); diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 98959e8cd448..42a2d0d3984a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -49,13 +49,16 @@ #define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS +#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ + KVM_DIRTY_LOG_INITIALLY_SET) + /* x86-specific vcpu->requests bit members */ #define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0) #define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1) #define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2) #define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3) #define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4) -#define KVM_REQ_LOAD_CR3 KVM_ARCH_REQ(5) +#define KVM_REQ_LOAD_MMU_PGD KVM_ARCH_REQ(5) #define KVM_REQ_EVENT KVM_ARCH_REQ(6) #define KVM_REQ_APF_HALT KVM_ARCH_REQ(7) #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8) @@ -182,7 +185,10 @@ enum exit_fastpath_completion { EXIT_FASTPATH_SKIP_EMUL_INS, }; -#include +struct x86_emulate_ctxt; +struct x86_exception; +enum x86_intercept; +enum x86_intercept_stage; #define KVM_NR_MEM_OBJS 40 @@ -297,7 +303,6 @@ union kvm_mmu_extended_role { unsigned int cr4_pke:1; unsigned int cr4_smap:1; unsigned int cr4_smep:1; - unsigned int cr4_la57:1; unsigned int maxphyaddr:6; }; }; @@ -382,8 +387,7 @@ struct kvm_mmu_root_info { * current mmu mode. */ struct kvm_mmu { - void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root); - unsigned long (*get_cr3)(struct kvm_vcpu *vcpu); + unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu); u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); int (*page_fault)(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u32 err, bool prefault); @@ -678,7 +682,7 @@ struct kvm_vcpu_arch { /* emulate context */ - struct x86_emulate_ctxt emulate_ctxt; + struct x86_emulate_ctxt *emulate_ctxt; bool emulate_regs_need_sync_to_vcpu; bool emulate_regs_need_sync_from_vcpu; int (*complete_userspace_io)(struct kvm_vcpu *vcpu); @@ -808,10 +812,6 @@ struct kvm_vcpu_arch { int pending_ioapic_eoi; int pending_external_vector; - /* GPA available */ - bool gpa_available; - gpa_t gpa_val; - /* be preempted when it's in kernel-mode(cpl=0) */ bool preempted_in_kernel; @@ -890,6 +890,7 @@ enum kvm_irqchip_mode { #define APICV_INHIBIT_REASON_NESTED 2 #define APICV_INHIBIT_REASON_IRQWIN 3 #define APICV_INHIBIT_REASON_PIT_REINJ 4 +#define APICV_INHIBIT_REASON_X2APIC 5 struct kvm_arch { unsigned long n_used_mmu_pages; @@ -920,6 +921,7 @@ struct kvm_arch { atomic_t vapics_in_nmi_mode; struct mutex apic_map_lock; struct kvm_apic_map *apic_map; + bool apic_map_dirty; bool apic_access_page_done; unsigned long apicv_inhibit_reasons; @@ -1052,19 +1054,14 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) } struct kvm_x86_ops { - int (*cpu_has_kvm_support)(void); /* __init */ - int (*disabled_by_bios)(void); /* __init */ int (*hardware_enable)(void); void (*hardware_disable)(void); - int (*check_processor_compatibility)(void);/* __init */ - int (*hardware_setup)(void); /* __init */ - void (*hardware_unsetup)(void); /* __exit */ + void (*hardware_unsetup)(void); bool (*cpu_has_accelerated_tpr)(void); bool (*has_emulated_msr)(int index); void (*cpuid_update)(struct kvm_vcpu *vcpu); - struct kvm *(*vm_alloc)(void); - void (*vm_free)(struct kvm *); + unsigned int vm_size; int (*vm_init)(struct kvm *kvm); void (*vm_destroy)(struct kvm *kvm); @@ -1090,7 +1087,6 @@ struct kvm_x86_ops { void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); - void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt); @@ -1153,13 +1149,8 @@ struct kvm_x86_ops { int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr); int (*get_tdp_level)(struct kvm_vcpu *vcpu); u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio); - int (*get_lpage_level)(void); - bool (*rdtscp_supported)(void); - bool (*invpcid_supported)(void); - void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); - - void (*set_supported_cpuid)(u32 func, struct kvm_cpuid_entry2 *entry); + void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, unsigned long cr3); bool (*has_wbinvd_exit)(void); @@ -1171,16 +1162,12 @@ struct kvm_x86_ops { int (*check_intercept)(struct kvm_vcpu *vcpu, struct x86_instruction_info *info, - enum x86_intercept_stage stage); + enum x86_intercept_stage stage, + struct x86_exception *exception); void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu, enum exit_fastpath_completion *exit_fastpath); - bool (*mpx_supported)(void); - bool (*xsaves_supported)(void); - bool (*umip_emulated)(void); - bool (*pt_supported)(void); - bool (*pku_supported)(void); - int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr); + int (*check_nested_events)(struct kvm_vcpu *vcpu); void (*request_immediate_exit)(struct kvm_vcpu *vcpu); void (*sched_in)(struct kvm_vcpu *kvm, int cpu); @@ -1269,6 +1256,15 @@ struct kvm_x86_ops { int (*enable_direct_tlbflush)(struct kvm_vcpu *vcpu); }; +struct kvm_x86_init_ops { + int (*cpu_has_kvm_support)(void); + int (*disabled_by_bios)(void); + int (*check_processor_compatibility)(void); + int (*hardware_setup)(void); + + struct kvm_x86_ops *runtime_ops; +}; + struct kvm_arch_async_pf { u32 token; gfn_t gfn; @@ -1276,25 +1272,24 @@ struct kvm_arch_async_pf { bool direct_map; }; -extern struct kvm_x86_ops *kvm_x86_ops; +extern u64 __read_mostly host_efer; + +extern struct kvm_x86_ops kvm_x86_ops; extern struct kmem_cache *x86_fpu_cache; #define __KVM_HAVE_ARCH_VM_ALLOC static inline struct kvm *kvm_arch_alloc_vm(void) { - return kvm_x86_ops->vm_alloc(); -} - -static inline void kvm_arch_free_vm(struct kvm *kvm) -{ - return kvm_x86_ops->vm_free(kvm); + return __vmalloc(kvm_x86_ops.vm_size, + GFP_KERNEL_ACCOUNT | __GFP_ZERO, PAGE_KERNEL); } +void kvm_arch_free_vm(struct kvm *kvm); #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm) { - if (kvm_x86_ops->tlb_remote_flush && - !kvm_x86_ops->tlb_remote_flush(kvm)) + if (kvm_x86_ops.tlb_remote_flush && + !kvm_x86_ops.tlb_remote_flush(kvm)) return 0; else return -ENOTSUPP; @@ -1313,7 +1308,8 @@ void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask, void kvm_mmu_reset_context(struct kvm_vcpu *vcpu); void kvm_mmu_slot_remove_write_access(struct kvm *kvm, - struct kvm_memory_slot *memslot); + struct kvm_memory_slot *memslot, + int start_level); void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, const struct kvm_memory_slot *memslot); void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, @@ -1379,10 +1375,11 @@ extern u64 kvm_mce_cap_supported; * * EMULTYPE_SKIP - Set when emulating solely to skip an instruction, i.e. to * decode the instruction length. For use *only* by - * kvm_x86_ops->skip_emulated_instruction() implementations. + * kvm_x86_ops.skip_emulated_instruction() implementations. * - * EMULTYPE_ALLOW_RETRY - Set when the emulator should resume the guest to - * retry native execution under certain conditions. + * EMULTYPE_ALLOW_RETRY_PF - Set when the emulator should resume the guest to + * retry native execution under certain conditions, + * Can only be set in conjunction with EMULTYPE_PF. * * EMULTYPE_TRAP_UD_FORCED - Set when emulating an intercepted #UD that was * triggered by KVM's magic "force emulation" prefix, @@ -1395,13 +1392,18 @@ extern u64 kvm_mce_cap_supported; * backdoor emulation, which is opt in via module param. * VMware backoor emulation handles select instructions * and reinjects the #GP for all other cases. + * + * EMULTYPE_PF - Set when emulating MMIO by way of an intercepted #PF, in which + * case the CR2/GPA value pass on the stack is valid. */ #define EMULTYPE_NO_DECODE (1 << 0) #define EMULTYPE_TRAP_UD (1 << 1) #define EMULTYPE_SKIP (1 << 2) -#define EMULTYPE_ALLOW_RETRY (1 << 3) +#define EMULTYPE_ALLOW_RETRY_PF (1 << 3) #define EMULTYPE_TRAP_UD_FORCED (1 << 4) #define EMULTYPE_VMWARE_GP (1 << 5) +#define EMULTYPE_PF (1 << 6) + int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type); int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, void *insn, int insn_len); @@ -1414,8 +1416,6 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data); int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu); int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu); -struct x86_emulate_ctxt; - int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in); int kvm_emulate_cpuid(struct kvm_vcpu *vcpu); int kvm_emulate_halt(struct kvm_vcpu *vcpu); @@ -1512,8 +1512,7 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush); -void kvm_enable_tdp(void); -void kvm_disable_tdp(void); +void kvm_configure_mmu(bool enable_tdp, int tdp_page_level); static inline gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, struct x86_exception *exception) @@ -1670,14 +1669,14 @@ static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq) static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) { - if (kvm_x86_ops->vcpu_blocking) - kvm_x86_ops->vcpu_blocking(vcpu); + if (kvm_x86_ops.vcpu_blocking) + kvm_x86_ops.vcpu_blocking(vcpu); } static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) { - if (kvm_x86_ops->vcpu_unblocking) - kvm_x86_ops->vcpu_unblocking(vcpu); + if (kvm_x86_ops.vcpu_unblocking) + kvm_x86_ops.vcpu_unblocking(vcpu); } static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h index 172f9749dbb2..87bd6025d91d 100644 --- a/arch/x86/include/asm/kvm_page_track.h +++ b/arch/x86/include/asm/kvm_page_track.h @@ -49,8 +49,7 @@ struct kvm_page_track_notifier_node { void kvm_page_track_init(struct kvm *kvm); void kvm_page_track_cleanup(struct kvm *kvm); -void kvm_page_track_free_memslot(struct kvm_memory_slot *free, - struct kvm_memory_slot *dont); +void kvm_page_track_free_memslot(struct kvm_memory_slot *slot); int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, unsigned long npages); diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index b538d9ddee9c..4e55370e48e8 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -213,21 +213,6 @@ static inline void arch_unmap(struct mm_struct *mm, unsigned long start, * So do not enforce things if the VMA is not from the current * mm, or if we are in a kernel thread. */ -static inline bool vma_is_foreign(struct vm_area_struct *vma) -{ - if (!current->mm) - return true; - /* - * Should PKRU be enforced on the access to this VMA? If - * the VMA is from another process, then PKRU has no - * relevance and should not be enforced. - */ - if (current->mm != vma->vm_mm) - return true; - - return false; -} - static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, bool execute, bool foreign) { diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 8521af3fef27..5e090d1f03f8 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -500,6 +500,18 @@ enum vmcs_field { VMX_EPT_EXECUTABLE_MASK) #define VMX_EPT_MT_MASK (7ull << VMX_EPT_MT_EPTE_SHIFT) +static inline u8 vmx_eptp_page_walk_level(u64 eptp) +{ + u64 encoded_level = eptp & VMX_EPTP_PWL_MASK; + + if (encoded_level == VMX_EPTP_PWL_5) + return 5; + + /* @eptp must be pre-validated by the caller. */ + WARN_ON_ONCE(encoded_level != VMX_EPTP_PWL_4); + return 4; +} + /* The mask to use to trigger an EPT Misconfiguration in order to track MMIO */ #define VMX_EPT_MISCONFIG_WX_VALUE (VMX_EPT_WRITABLE_MASK | \ VMX_EPT_EXECUTABLE_MASK) diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index bb5abfef0256..ba89cabe5fcf 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -155,6 +155,4 @@ ifeq ($(CONFIG_X86_64),y) obj-y += vsmp_64.o endif -ifdef CONFIG_EFI -obj-$(CONFIG_IMA) += ima_arch.o -endif +obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT) += ima_arch.o diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index b1c469446b07..901cd1fdecd9 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -24,6 +24,13 @@ #include "trace.h" #include "pmu.h" +/* + * Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be + * aligned to sizeof(unsigned long) because it's not accessed via bitops. + */ +u32 kvm_cpu_caps[NCAPINTS] __read_mostly; +EXPORT_SYMBOL_GPL(kvm_cpu_caps); + static u32 xstate_required_size(u64 xstate_bv, bool compacted) { int feature_bit = 0; @@ -45,23 +52,6 @@ static u32 xstate_required_size(u64 xstate_bv, bool compacted) return ret; } -bool kvm_mpx_supported(void) -{ - return ((host_xcr0 & (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR)) - && kvm_x86_ops->mpx_supported()); -} -EXPORT_SYMBOL_GPL(kvm_mpx_supported); - -u64 kvm_supported_xcr0(void) -{ - u64 xcr0 = KVM_SUPPORTED_XCR0 & host_xcr0; - - if (!kvm_mpx_supported()) - xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); - - return xcr0; -} - #define F feature_bit int kvm_update_cpuid(struct kvm_vcpu *vcpu) @@ -74,32 +64,24 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu) return 0; /* Update OSXSAVE bit */ - if (boot_cpu_has(X86_FEATURE_XSAVE) && best->function == 0x1) { - best->ecx &= ~F(OSXSAVE); - if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) - best->ecx |= F(OSXSAVE); - } + if (boot_cpu_has(X86_FEATURE_XSAVE) && best->function == 0x1) + cpuid_entry_change(best, X86_FEATURE_OSXSAVE, + kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)); - best->edx &= ~F(APIC); - if (vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE) - best->edx |= F(APIC); + cpuid_entry_change(best, X86_FEATURE_APIC, + vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE); if (apic) { - if (best->ecx & F(TSC_DEADLINE_TIMER)) + if (cpuid_entry_has(best, X86_FEATURE_TSC_DEADLINE_TIMER)) apic->lapic_timer.timer_mode_mask = 3 << 17; else apic->lapic_timer.timer_mode_mask = 1 << 17; } best = kvm_find_cpuid_entry(vcpu, 7, 0); - if (best) { - /* Update OSPKE bit */ - if (boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7) { - best->ecx &= ~F(OSPKE); - if (kvm_read_cr4_bits(vcpu, X86_CR4_PKE)) - best->ecx |= F(OSPKE); - } - } + if (best && boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7) + cpuid_entry_change(best, X86_FEATURE_OSPKE, + kvm_read_cr4_bits(vcpu, X86_CR4_PKE)); best = kvm_find_cpuid_entry(vcpu, 0xD, 0); if (!best) { @@ -107,14 +89,14 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu) vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; } else { vcpu->arch.guest_supported_xcr0 = - (best->eax | ((u64)best->edx << 32)) & - kvm_supported_xcr0(); + (best->eax | ((u64)best->edx << 32)) & supported_xcr0; vcpu->arch.guest_xstate_size = best->ebx = xstate_required_size(vcpu->arch.xcr0, false); } best = kvm_find_cpuid_entry(vcpu, 0xD, 1); - if (best && (best->eax & (F(XSAVES) | F(XSAVEC)))) + if (best && (cpuid_entry_has(best, X86_FEATURE_XSAVES) || + cpuid_entry_has(best, X86_FEATURE_XSAVEC))) best->ebx = xstate_required_size(vcpu->arch.xcr0, true); /* @@ -136,12 +118,10 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu) if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT)) { best = kvm_find_cpuid_entry(vcpu, 0x1, 0); - if (best) { - if (vcpu->arch.ia32_misc_enable_msr & MSR_IA32_MISC_ENABLE_MWAIT) - best->ecx |= F(MWAIT); - else - best->ecx &= ~F(MWAIT); - } + if (best) + cpuid_entry_change(best, X86_FEATURE_MWAIT, + vcpu->arch.ia32_misc_enable_msr & + MSR_IA32_MISC_ENABLE_MWAIT); } /* Update physical-address width */ @@ -154,10 +134,7 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu) static int is_efer_nx(void) { - unsigned long long efer = 0; - - rdmsrl_safe(MSR_EFER, &efer); - return efer & EFER_NX; + return host_efer & EFER_NX; } static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) @@ -173,8 +150,8 @@ static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) break; } } - if (entry && (entry->edx & F(NX)) && !is_efer_nx()) { - entry->edx &= ~F(NX); + if (entry && cpuid_entry_has(entry, X86_FEATURE_NX) && !is_efer_nx()) { + cpuid_entry_clear(entry, X86_FEATURE_NX); printk(KERN_INFO "kvm: guest NX capability removed\n"); } } @@ -232,7 +209,7 @@ int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, vcpu->arch.cpuid_nent = cpuid->nent; cpuid_fix_nx_cap(vcpu); kvm_apic_set_version(vcpu); - kvm_x86_ops->cpuid_update(vcpu); + kvm_x86_ops.cpuid_update(vcpu); r = kvm_update_cpuid(vcpu); out: @@ -255,7 +232,7 @@ int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, goto out; vcpu->arch.cpuid_nent = cpuid->nent; kvm_apic_set_version(vcpu); - kvm_x86_ops->cpuid_update(vcpu); + kvm_x86_ops.cpuid_update(vcpu); r = kvm_update_cpuid(vcpu); out: return r; @@ -281,15 +258,189 @@ int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, return r; } -static __always_inline void cpuid_mask(u32 *word, int wordnum) +static __always_inline void kvm_cpu_cap_mask(enum cpuid_leafs leaf, u32 mask) { - reverse_cpuid_check(wordnum); - *word &= boot_cpu_data.x86_capability[wordnum]; + const struct cpuid_reg cpuid = x86_feature_cpuid(leaf * 32); + struct kvm_cpuid_entry2 entry; + + reverse_cpuid_check(leaf); + kvm_cpu_caps[leaf] &= mask; + + cpuid_count(cpuid.function, cpuid.index, + &entry.eax, &entry.ebx, &entry.ecx, &entry.edx); + + kvm_cpu_caps[leaf] &= *__cpuid_entry_get_reg(&entry, cpuid.reg); } -static void do_host_cpuid(struct kvm_cpuid_entry2 *entry, u32 function, - u32 index) +void kvm_set_cpu_caps(void) { + unsigned int f_nx = is_efer_nx() ? F(NX) : 0; +#ifdef CONFIG_X86_64 + unsigned int f_gbpages = F(GBPAGES); + unsigned int f_lm = F(LM); +#else + unsigned int f_gbpages = 0; + unsigned int f_lm = 0; +#endif + + BUILD_BUG_ON(sizeof(kvm_cpu_caps) > + sizeof(boot_cpu_data.x86_capability)); + + memcpy(&kvm_cpu_caps, &boot_cpu_data.x86_capability, + sizeof(kvm_cpu_caps)); + + kvm_cpu_cap_mask(CPUID_1_ECX, + /* + * NOTE: MONITOR (and MWAIT) are emulated as NOP, but *not* + * advertised to guests via CPUID! + */ + F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ | + 0 /* DS-CPL, VMX, SMX, EST */ | + 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ | + F(FMA) | F(CX16) | 0 /* xTPR Update, PDCM */ | + F(PCID) | 0 /* Reserved, DCA */ | F(XMM4_1) | + F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) | + 0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) | + F(F16C) | F(RDRAND) + ); + /* KVM emulates x2apic in software irrespective of host support. */ + kvm_cpu_cap_set(X86_FEATURE_X2APIC); + + kvm_cpu_cap_mask(CPUID_1_EDX, + F(FPU) | F(VME) | F(DE) | F(PSE) | + F(TSC) | F(MSR) | F(PAE) | F(MCE) | + F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) | + F(MTRR) | F(PGE) | F(MCA) | F(CMOV) | + F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLUSH) | + 0 /* Reserved, DS, ACPI */ | F(MMX) | + F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) | + 0 /* HTT, TM, Reserved, PBE */ + ); + + kvm_cpu_cap_mask(CPUID_7_0_EBX, + F(FSGSBASE) | F(BMI1) | F(HLE) | F(AVX2) | F(SMEP) | + F(BMI2) | F(ERMS) | 0 /*INVPCID*/ | F(RTM) | 0 /*MPX*/ | F(RDSEED) | + F(ADX) | F(SMAP) | F(AVX512IFMA) | F(AVX512F) | F(AVX512PF) | + F(AVX512ER) | F(AVX512CD) | F(CLFLUSHOPT) | F(CLWB) | F(AVX512DQ) | + F(SHA_NI) | F(AVX512BW) | F(AVX512VL) | 0 /*INTEL_PT*/ + ); + + kvm_cpu_cap_mask(CPUID_7_ECX, + F(AVX512VBMI) | F(LA57) | 0 /*PKU*/ | 0 /*OSPKE*/ | F(RDPID) | + F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) | + F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) | + F(CLDEMOTE) | F(MOVDIRI) | F(MOVDIR64B) | 0 /*WAITPKG*/ + ); + /* Set LA57 based on hardware capability. */ + if (cpuid_ecx(7) & F(LA57)) + kvm_cpu_cap_set(X86_FEATURE_LA57); + + kvm_cpu_cap_mask(CPUID_7_EDX, + F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) | + F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) | + F(MD_CLEAR) | F(AVX512_VP2INTERSECT) | F(FSRM) + ); + + /* TSC_ADJUST and ARCH_CAPABILITIES are emulated in software. */ + kvm_cpu_cap_set(X86_FEATURE_TSC_ADJUST); + kvm_cpu_cap_set(X86_FEATURE_ARCH_CAPABILITIES); + + if (boot_cpu_has(X86_FEATURE_IBPB) && boot_cpu_has(X86_FEATURE_IBRS)) + kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL); + if (boot_cpu_has(X86_FEATURE_STIBP)) + kvm_cpu_cap_set(X86_FEATURE_INTEL_STIBP); + if (boot_cpu_has(X86_FEATURE_AMD_SSBD)) + kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL_SSBD); + + kvm_cpu_cap_mask(CPUID_7_1_EAX, + F(AVX512_BF16) + ); + + kvm_cpu_cap_mask(CPUID_D_1_EAX, + F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | F(XSAVES) + ); + + kvm_cpu_cap_mask(CPUID_8000_0001_ECX, + F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ | + F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) | + F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) | + 0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) | + F(TOPOEXT) | F(PERFCTR_CORE) + ); + + kvm_cpu_cap_mask(CPUID_8000_0001_EDX, + F(FPU) | F(VME) | F(DE) | F(PSE) | + F(TSC) | F(MSR) | F(PAE) | F(MCE) | + F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) | + F(MTRR) | F(PGE) | F(MCA) | F(CMOV) | + F(PAT) | F(PSE36) | 0 /* Reserved */ | + f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) | + F(FXSR) | F(FXSR_OPT) | f_gbpages | F(RDTSCP) | + 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW) + ); + + if (!tdp_enabled && IS_ENABLED(CONFIG_X86_64)) + kvm_cpu_cap_set(X86_FEATURE_GBPAGES); + + kvm_cpu_cap_mask(CPUID_8000_0008_EBX, + F(CLZERO) | F(XSAVEERPTR) | + F(WBNOINVD) | F(AMD_IBPB) | F(AMD_IBRS) | F(AMD_SSBD) | F(VIRT_SSBD) | + F(AMD_SSB_NO) | F(AMD_STIBP) | F(AMD_STIBP_ALWAYS_ON) + ); + + /* + * AMD has separate bits for each SPEC_CTRL bit. + * arch/x86/kernel/cpu/bugs.c is kind enough to + * record that in cpufeatures so use them. + */ + if (boot_cpu_has(X86_FEATURE_IBPB)) + kvm_cpu_cap_set(X86_FEATURE_AMD_IBPB); + if (boot_cpu_has(X86_FEATURE_IBRS)) + kvm_cpu_cap_set(X86_FEATURE_AMD_IBRS); + if (boot_cpu_has(X86_FEATURE_STIBP)) + kvm_cpu_cap_set(X86_FEATURE_AMD_STIBP); + if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD)) + kvm_cpu_cap_set(X86_FEATURE_AMD_SSBD); + if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) + kvm_cpu_cap_set(X86_FEATURE_AMD_SSB_NO); + /* + * The preference is to use SPEC CTRL MSR instead of the + * VIRT_SPEC MSR. + */ + if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) && + !boot_cpu_has(X86_FEATURE_AMD_SSBD)) + kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); + + /* + * Hide all SVM features by default, SVM will set the cap bits for + * features it emulates and/or exposes for L1. + */ + kvm_cpu_cap_mask(CPUID_8000_000A_EDX, 0); + + kvm_cpu_cap_mask(CPUID_C000_0001_EDX, + F(XSTORE) | F(XSTORE_EN) | F(XCRYPT) | F(XCRYPT_EN) | + F(ACE2) | F(ACE2_EN) | F(PHE) | F(PHE_EN) | + F(PMM) | F(PMM_EN) + ); +} +EXPORT_SYMBOL_GPL(kvm_set_cpu_caps); + +struct kvm_cpuid_array { + struct kvm_cpuid_entry2 *entries; + const int maxnent; + int nent; +}; + +static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array, + u32 function, u32 index) +{ + struct kvm_cpuid_entry2 *entry; + + if (array->nent >= array->maxnent) + return NULL; + + entry = &array->entries[array->nent++]; + entry->function = function; entry->index = index; entry->flags = 0; @@ -298,9 +449,6 @@ static void do_host_cpuid(struct kvm_cpuid_entry2 *entry, u32 function, &entry->eax, &entry->ebx, &entry->ecx, &entry->edx); switch (function) { - case 2: - entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; - break; case 4: case 7: case 0xb: @@ -316,11 +464,18 @@ static void do_host_cpuid(struct kvm_cpuid_entry2 *entry, u32 function, entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; break; } + + return entry; } -static int __do_cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, - u32 func, int *nent, int maxnent) +static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func) { + struct kvm_cpuid_entry2 *entry; + + if (array->nent >= array->maxnent) + return -E2BIG; + + entry = &array->entries[array->nent]; entry->function = func; entry->index = 0; entry->flags = 0; @@ -328,17 +483,17 @@ static int __do_cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, switch (func) { case 0: entry->eax = 7; - ++*nent; + ++array->nent; break; case 1: entry->ecx = F(MOVBE); - ++*nent; + ++array->nent; break; case 7: entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; entry->eax = 0; entry->ecx = F(RDPID); - ++*nent; + ++array->nent; default: break; } @@ -346,223 +501,60 @@ static int __do_cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, return 0; } -static inline void do_cpuid_7_mask(struct kvm_cpuid_entry2 *entry, int index) +static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) { - unsigned f_invpcid = kvm_x86_ops->invpcid_supported() ? F(INVPCID) : 0; - unsigned f_mpx = kvm_mpx_supported() ? F(MPX) : 0; - unsigned f_umip = kvm_x86_ops->umip_emulated() ? F(UMIP) : 0; - unsigned f_intel_pt = kvm_x86_ops->pt_supported() ? F(INTEL_PT) : 0; - unsigned f_la57; - unsigned f_pku = kvm_x86_ops->pku_supported() ? F(PKU) : 0; - - /* cpuid 7.0.ebx */ - const u32 kvm_cpuid_7_0_ebx_x86_features = - F(FSGSBASE) | F(BMI1) | F(HLE) | F(AVX2) | F(SMEP) | - F(BMI2) | F(ERMS) | f_invpcid | F(RTM) | f_mpx | F(RDSEED) | - F(ADX) | F(SMAP) | F(AVX512IFMA) | F(AVX512F) | F(AVX512PF) | - F(AVX512ER) | F(AVX512CD) | F(CLFLUSHOPT) | F(CLWB) | F(AVX512DQ) | - F(SHA_NI) | F(AVX512BW) | F(AVX512VL) | f_intel_pt; - - /* cpuid 7.0.ecx*/ - const u32 kvm_cpuid_7_0_ecx_x86_features = - F(AVX512VBMI) | F(LA57) | 0 /*PKU*/ | 0 /*OSPKE*/ | F(RDPID) | - F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) | - F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) | - F(CLDEMOTE) | F(MOVDIRI) | F(MOVDIR64B) | 0 /*WAITPKG*/; - - /* cpuid 7.0.edx*/ - const u32 kvm_cpuid_7_0_edx_x86_features = - F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) | - F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) | - F(MD_CLEAR); - - /* cpuid 7.1.eax */ - const u32 kvm_cpuid_7_1_eax_x86_features = - F(AVX512_BF16); - - switch (index) { - case 0: - entry->eax = min(entry->eax, 1u); - entry->ebx &= kvm_cpuid_7_0_ebx_x86_features; - cpuid_mask(&entry->ebx, CPUID_7_0_EBX); - /* TSC_ADJUST is emulated */ - entry->ebx |= F(TSC_ADJUST); - - entry->ecx &= kvm_cpuid_7_0_ecx_x86_features; - f_la57 = entry->ecx & F(LA57); - cpuid_mask(&entry->ecx, CPUID_7_ECX); - /* Set LA57 based on hardware capability. */ - entry->ecx |= f_la57; - entry->ecx |= f_umip; - entry->ecx |= f_pku; - /* PKU is not yet implemented for shadow paging. */ - if (!tdp_enabled || !boot_cpu_has(X86_FEATURE_OSPKE)) - entry->ecx &= ~F(PKU); - - entry->edx &= kvm_cpuid_7_0_edx_x86_features; - cpuid_mask(&entry->edx, CPUID_7_EDX); - if (boot_cpu_has(X86_FEATURE_IBPB) && boot_cpu_has(X86_FEATURE_IBRS)) - entry->edx |= F(SPEC_CTRL); - if (boot_cpu_has(X86_FEATURE_STIBP)) - entry->edx |= F(INTEL_STIBP); - if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || - boot_cpu_has(X86_FEATURE_AMD_SSBD)) - entry->edx |= F(SPEC_CTRL_SSBD); - /* - * We emulate ARCH_CAPABILITIES in software even - * if the host doesn't support it. - */ - entry->edx |= F(ARCH_CAPABILITIES); - break; - case 1: - entry->eax &= kvm_cpuid_7_1_eax_x86_features; - entry->ebx = 0; - entry->ecx = 0; - entry->edx = 0; - break; - default: - WARN_ON_ONCE(1); - entry->eax = 0; - entry->ebx = 0; - entry->ecx = 0; - entry->edx = 0; - break; - } -} - -static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, - int *nent, int maxnent) -{ - int r; - unsigned f_nx = is_efer_nx() ? F(NX) : 0; -#ifdef CONFIG_X86_64 - unsigned f_gbpages = (kvm_x86_ops->get_lpage_level() == PT_PDPE_LEVEL) - ? F(GBPAGES) : 0; - unsigned f_lm = F(LM); -#else - unsigned f_gbpages = 0; - unsigned f_lm = 0; -#endif - unsigned f_rdtscp = kvm_x86_ops->rdtscp_supported() ? F(RDTSCP) : 0; - unsigned f_xsaves = kvm_x86_ops->xsaves_supported() ? F(XSAVES) : 0; - unsigned f_intel_pt = kvm_x86_ops->pt_supported() ? F(INTEL_PT) : 0; - - /* cpuid 1.edx */ - const u32 kvm_cpuid_1_edx_x86_features = - F(FPU) | F(VME) | F(DE) | F(PSE) | - F(TSC) | F(MSR) | F(PAE) | F(MCE) | - F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) | - F(MTRR) | F(PGE) | F(MCA) | F(CMOV) | - F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLUSH) | - 0 /* Reserved, DS, ACPI */ | F(MMX) | - F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) | - 0 /* HTT, TM, Reserved, PBE */; - /* cpuid 0x80000001.edx */ - const u32 kvm_cpuid_8000_0001_edx_x86_features = - F(FPU) | F(VME) | F(DE) | F(PSE) | - F(TSC) | F(MSR) | F(PAE) | F(MCE) | - F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) | - F(MTRR) | F(PGE) | F(MCA) | F(CMOV) | - F(PAT) | F(PSE36) | 0 /* Reserved */ | - f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) | - F(FXSR) | F(FXSR_OPT) | f_gbpages | f_rdtscp | - 0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW); - /* cpuid 1.ecx */ - const u32 kvm_cpuid_1_ecx_x86_features = - /* NOTE: MONITOR (and MWAIT) are emulated as NOP, - * but *not* advertised to guests via CPUID ! */ - F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ | - 0 /* DS-CPL, VMX, SMX, EST */ | - 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ | - F(FMA) | F(CX16) | 0 /* xTPR Update, PDCM */ | - F(PCID) | 0 /* Reserved, DCA */ | F(XMM4_1) | - F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) | - 0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) | - F(F16C) | F(RDRAND); - /* cpuid 0x80000001.ecx */ - const u32 kvm_cpuid_8000_0001_ecx_x86_features = - F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ | - F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) | - F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) | - 0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) | - F(TOPOEXT) | F(PERFCTR_CORE); - - /* cpuid 0x80000008.ebx */ - const u32 kvm_cpuid_8000_0008_ebx_x86_features = - F(CLZERO) | F(XSAVEERPTR) | - F(WBNOINVD) | F(AMD_IBPB) | F(AMD_IBRS) | F(AMD_SSBD) | F(VIRT_SSBD) | - F(AMD_SSB_NO) | F(AMD_STIBP) | F(AMD_STIBP_ALWAYS_ON); - - /* cpuid 0xC0000001.edx */ - const u32 kvm_cpuid_C000_0001_edx_x86_features = - F(XSTORE) | F(XSTORE_EN) | F(XCRYPT) | F(XCRYPT_EN) | - F(ACE2) | F(ACE2_EN) | F(PHE) | F(PHE_EN) | - F(PMM) | F(PMM_EN); - - /* cpuid 0xD.1.eax */ - const u32 kvm_cpuid_D_1_eax_x86_features = - F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | f_xsaves; + struct kvm_cpuid_entry2 *entry; + int r, i, max_idx; /* all calls to cpuid_count() should be made on the same cpu */ get_cpu(); r = -E2BIG; - if (WARN_ON(*nent >= maxnent)) + entry = do_host_cpuid(array, function, 0); + if (!entry) goto out; - do_host_cpuid(entry, function, 0); - ++*nent; - switch (function) { case 0: /* Limited to the highest leaf implemented in KVM. */ entry->eax = min(entry->eax, 0x1fU); break; case 1: - entry->edx &= kvm_cpuid_1_edx_x86_features; - cpuid_mask(&entry->edx, CPUID_1_EDX); - entry->ecx &= kvm_cpuid_1_ecx_x86_features; - cpuid_mask(&entry->ecx, CPUID_1_ECX); - /* we support x2apic emulation even if host does not support - * it since we emulate x2apic in software */ - entry->ecx |= F(X2APIC); + cpuid_entry_override(entry, CPUID_1_EDX); + cpuid_entry_override(entry, CPUID_1_ECX); break; - /* function 2 entries are STATEFUL. That is, repeated cpuid commands - * may return different values. This forces us to get_cpu() before - * issuing the first command, and also to emulate this annoying behavior - * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */ - case 2: { - int t, times = entry->eax & 0xff; - - entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; - for (t = 1; t < times; ++t) { - if (*nent >= maxnent) - goto out; - - do_host_cpuid(&entry[t], function, 0); - ++*nent; - } + case 2: + /* + * On ancient CPUs, function 2 entries are STATEFUL. That is, + * CPUID(function=2, index=0) may return different results each + * time, with the least-significant byte in EAX enumerating the + * number of times software should do CPUID(2, 0). + * + * Modern CPUs, i.e. every CPU KVM has *ever* run on are less + * idiotic. Intel's SDM states that EAX & 0xff "will always + * return 01H. Software should ignore this value and not + * interpret it as an informational descriptor", while AMD's + * APM states that CPUID(2) is reserved. + * + * WARN if a frankenstein CPU that supports virtualization and + * a stateful CPUID.0x2 is encountered. + */ + WARN_ON_ONCE((entry->eax & 0xff) > 1); break; - } /* functions 4 and 0x8000001d have additional index. */ case 4: - case 0x8000001d: { - int i, cache_type; - - /* read more entries until cache_type is zero */ - for (i = 1; ; ++i) { - if (*nent >= maxnent) + case 0x8000001d: + /* + * Read entries until the cache type in the previous entry is + * zero, i.e. indicates an invalid entry. + */ + for (i = 1; entry->eax & 0x1f; ++i) { + entry = do_host_cpuid(array, function, i); + if (!entry) goto out; - - cache_type = entry[i - 1].eax & 0x1f; - if (!cache_type) - break; - do_host_cpuid(&entry[i], function, i); - ++*nent; } break; - } case 6: /* Thermal management */ entry->eax = 0x4; /* allow ARAT */ entry->ebx = 0; @@ -570,22 +562,24 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, entry->edx = 0; break; /* function 7 has additional index. */ - case 7: { - int i; + case 7: + entry->eax = min(entry->eax, 1u); + cpuid_entry_override(entry, CPUID_7_0_EBX); + cpuid_entry_override(entry, CPUID_7_ECX); + cpuid_entry_override(entry, CPUID_7_EDX); - for (i = 0; ; ) { - do_cpuid_7_mask(&entry[i], i); - if (i == entry->eax) - break; - if (*nent >= maxnent) + /* KVM only supports 0x7.0 and 0x7.1, capped above via min(). */ + if (entry->eax == 1) { + entry = do_host_cpuid(array, function, 1); + if (!entry) goto out; - ++i; - do_host_cpuid(&entry[i], function, i); - ++*nent; + cpuid_entry_override(entry, CPUID_7_1_EAX); + entry->ebx = 0; + entry->ecx = 0; + entry->edx = 0; } break; - } case 9: break; case 0xa: { /* Architectural Performance Monitoring */ @@ -622,79 +616,81 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, * thus they can be handled by common code. */ case 0x1f: - case 0xb: { - int i; - + case 0xb: /* - * We filled in entry[0] for CPUID(EAX=, - * ECX=00H) above. If its level type (ECX[15:8]) is - * zero, then the leaf is unimplemented, and we're - * done. Otherwise, continue to populate entries - * until the level type (ECX[15:8]) of the previously - * added entry is zero. + * Populate entries until the level type (ECX[15:8]) of the + * previous entry is zero. Note, CPUID EAX.{0x1f,0xb}.0 is + * the starting entry, filled by the primary do_host_cpuid(). */ - for (i = 1; entry[i - 1].ecx & 0xff00; ++i) { - if (*nent >= maxnent) + for (i = 1; entry->ecx & 0xff00; ++i) { + entry = do_host_cpuid(array, function, i); + if (!entry) goto out; - - do_host_cpuid(&entry[i], function, i); - ++*nent; } break; - } - case 0xd: { - int idx, i; - u64 supported = kvm_supported_xcr0(); - - entry->eax &= supported; - entry->ebx = xstate_required_size(supported, false); + case 0xd: + entry->eax &= supported_xcr0; + entry->ebx = xstate_required_size(supported_xcr0, false); entry->ecx = entry->ebx; - entry->edx &= supported >> 32; - if (!supported) + entry->edx &= supported_xcr0 >> 32; + if (!supported_xcr0) break; - for (idx = 1, i = 1; idx < 64; ++idx) { - u64 mask = ((u64)1 << idx); - if (*nent >= maxnent) + entry = do_host_cpuid(array, function, 1); + if (!entry) + goto out; + + cpuid_entry_override(entry, CPUID_D_1_EAX); + if (entry->eax & (F(XSAVES)|F(XSAVEC))) + entry->ebx = xstate_required_size(supported_xcr0 | supported_xss, + true); + else { + WARN_ON_ONCE(supported_xss != 0); + entry->ebx = 0; + } + entry->ecx &= supported_xss; + entry->edx &= supported_xss >> 32; + + for (i = 2; i < 64; ++i) { + bool s_state; + if (supported_xcr0 & BIT_ULL(i)) + s_state = false; + else if (supported_xss & BIT_ULL(i)) + s_state = true; + else + continue; + + entry = do_host_cpuid(array, function, i); + if (!entry) goto out; - do_host_cpuid(&entry[i], function, idx); - if (idx == 1) { - entry[i].eax &= kvm_cpuid_D_1_eax_x86_features; - cpuid_mask(&entry[i].eax, CPUID_D_1_EAX); - entry[i].ebx = 0; - if (entry[i].eax & (F(XSAVES)|F(XSAVEC))) - entry[i].ebx = - xstate_required_size(supported, - true); - } else { - if (entry[i].eax == 0 || !(supported & mask)) - continue; - if (WARN_ON_ONCE(entry[i].ecx & 1)) - continue; + /* + * The supported check above should have filtered out + * invalid sub-leafs. Only valid sub-leafs should + * reach this point, and they should have a non-zero + * save state size. Furthermore, check whether the + * processor agrees with supported_xcr0/supported_xss + * on whether this is an XCR0- or IA32_XSS-managed area. + */ + if (WARN_ON_ONCE(!entry->eax || (entry->ecx & 0x1) != s_state)) { + --array->nent; + continue; } - entry[i].ecx = 0; - entry[i].edx = 0; - ++*nent; - ++i; + entry->edx = 0; } break; - } /* Intel PT */ - case 0x14: { - int t, times = entry->eax; - - if (!f_intel_pt) + case 0x14: + if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) { + entry->eax = entry->ebx = entry->ecx = entry->edx = 0; break; + } - for (t = 1; t <= times; ++t) { - if (*nent >= maxnent) + for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) { + if (!do_host_cpuid(array, function, i)) goto out; - do_host_cpuid(&entry[t], function, t); - ++*nent; } break; - } case KVM_CPUID_SIGNATURE: { static const char signature[12] = "KVMKVMKVM\0\0"; const u32 *sigptr = (const u32 *)signature; @@ -729,10 +725,8 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, entry->eax = min(entry->eax, 0x8000001f); break; case 0x80000001: - entry->edx &= kvm_cpuid_8000_0001_edx_x86_features; - cpuid_mask(&entry->edx, CPUID_8000_0001_EDX); - entry->ecx &= kvm_cpuid_8000_0001_ecx_x86_features; - cpuid_mask(&entry->ecx, CPUID_8000_0001_ECX); + cpuid_entry_override(entry, CPUID_8000_0001_EDX); + cpuid_entry_override(entry, CPUID_8000_0001_ECX); break; case 0x80000007: /* Advanced power management */ /* invariant TSC is CPUID.80000007H:EDX[8] */ @@ -750,33 +744,20 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, g_phys_as = phys_as; entry->eax = g_phys_as | (virt_as << 8); entry->edx = 0; - entry->ebx &= kvm_cpuid_8000_0008_ebx_x86_features; - cpuid_mask(&entry->ebx, CPUID_8000_0008_EBX); - /* - * AMD has separate bits for each SPEC_CTRL bit. - * arch/x86/kernel/cpu/bugs.c is kind enough to - * record that in cpufeatures so use them. - */ - if (boot_cpu_has(X86_FEATURE_IBPB)) - entry->ebx |= F(AMD_IBPB); - if (boot_cpu_has(X86_FEATURE_IBRS)) - entry->ebx |= F(AMD_IBRS); - if (boot_cpu_has(X86_FEATURE_STIBP)) - entry->ebx |= F(AMD_STIBP); - if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || - boot_cpu_has(X86_FEATURE_AMD_SSBD)) - entry->ebx |= F(AMD_SSBD); - if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) - entry->ebx |= F(AMD_SSB_NO); - /* - * The preference is to use SPEC CTRL MSR instead of the - * VIRT_SPEC MSR. - */ - if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) && - !boot_cpu_has(X86_FEATURE_AMD_SSBD)) - entry->ebx |= F(VIRT_SSBD); + cpuid_entry_override(entry, CPUID_8000_0008_EBX); break; } + case 0x8000000A: + if (!kvm_cpu_cap_has(X86_FEATURE_SVM)) { + entry->eax = entry->ebx = entry->ecx = entry->edx = 0; + break; + } + entry->eax = 1; /* SVM revision 1 */ + entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper + ASID emulation to nested SVM */ + entry->ecx = 0; /* Reserved */ + cpuid_entry_override(entry, CPUID_8000_000A_EDX); + break; case 0x80000019: entry->ecx = entry->edx = 0; break; @@ -794,8 +775,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, entry->eax = min(entry->eax, 0xC0000004); break; case 0xC0000001: - entry->edx &= kvm_cpuid_C000_0001_edx_x86_features; - cpuid_mask(&entry->edx, CPUID_C000_0001_EDX); + cpuid_entry_override(entry, CPUID_C000_0001_EDX); break; case 3: /* Processor serial number */ case 5: /* MONITOR/MWAIT */ @@ -807,8 +787,6 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, break; } - kvm_x86_ops->set_supported_cpuid(function, entry); - r = 0; out: @@ -817,26 +795,39 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, return r; } -static int do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 func, - int *nent, int maxnent, unsigned int type) +static int do_cpuid_func(struct kvm_cpuid_array *array, u32 func, + unsigned int type) { - if (*nent >= maxnent) - return -E2BIG; - if (type == KVM_GET_EMULATED_CPUID) - return __do_cpuid_func_emulated(entry, func, nent, maxnent); + return __do_cpuid_func_emulated(array, func); - return __do_cpuid_func(entry, func, nent, maxnent); + return __do_cpuid_func(array, func); } -struct kvm_cpuid_param { - u32 func; - bool (*qualifier)(const struct kvm_cpuid_param *param); -}; +#define CENTAUR_CPUID_SIGNATURE 0xC0000000 -static bool is_centaur_cpu(const struct kvm_cpuid_param *param) +static int get_cpuid_func(struct kvm_cpuid_array *array, u32 func, + unsigned int type) { - return boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR; + u32 limit; + int r; + + if (func == CENTAUR_CPUID_SIGNATURE && + boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR) + return 0; + + r = do_cpuid_func(array, func, type); + if (r) + return r; + + limit = array->entries[array->nent - 1].eax; + for (func = func + 1; func <= limit; ++func) { + r = do_cpuid_func(array, func, type); + if (r) + break; + } + + return r; } static bool sanity_check_entries(struct kvm_cpuid_entry2 __user *entries, @@ -870,157 +861,145 @@ int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid, struct kvm_cpuid_entry2 __user *entries, unsigned int type) { - struct kvm_cpuid_entry2 *cpuid_entries; - int limit, nent = 0, r = -E2BIG, i; - u32 func; - static const struct kvm_cpuid_param param[] = { - { .func = 0 }, - { .func = 0x80000000 }, - { .func = 0xC0000000, .qualifier = is_centaur_cpu }, - { .func = KVM_CPUID_SIGNATURE }, + static const u32 funcs[] = { + 0, 0x80000000, CENTAUR_CPUID_SIGNATURE, KVM_CPUID_SIGNATURE, }; + struct kvm_cpuid_array array = { + .nent = 0, + .maxnent = cpuid->nent, + }; + int r, i; + if (cpuid->nent < 1) - goto out; + return -E2BIG; if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) cpuid->nent = KVM_MAX_CPUID_ENTRIES; if (sanity_check_entries(entries, cpuid->nent, type)) return -EINVAL; - r = -ENOMEM; - cpuid_entries = vzalloc(array_size(sizeof(struct kvm_cpuid_entry2), + array.entries = vzalloc(array_size(sizeof(struct kvm_cpuid_entry2), cpuid->nent)); - if (!cpuid_entries) - goto out; - - r = 0; - for (i = 0; i < ARRAY_SIZE(param); i++) { - const struct kvm_cpuid_param *ent = ¶m[i]; - - if (ent->qualifier && !ent->qualifier(ent)) - continue; - - r = do_cpuid_func(&cpuid_entries[nent], ent->func, - &nent, cpuid->nent, type); - - if (r) - goto out_free; - - limit = cpuid_entries[nent - 1].eax; - for (func = ent->func + 1; func <= limit && nent < cpuid->nent && r == 0; ++func) - r = do_cpuid_func(&cpuid_entries[nent], func, - &nent, cpuid->nent, type); + if (!array.entries) + return -ENOMEM; + for (i = 0; i < ARRAY_SIZE(funcs); i++) { + r = get_cpuid_func(&array, funcs[i], type); if (r) goto out_free; } + cpuid->nent = array.nent; - r = -EFAULT; - if (copy_to_user(entries, cpuid_entries, - nent * sizeof(struct kvm_cpuid_entry2))) - goto out_free; - cpuid->nent = nent; - r = 0; + if (copy_to_user(entries, array.entries, + array.nent * sizeof(struct kvm_cpuid_entry2))) + r = -EFAULT; out_free: - vfree(cpuid_entries); -out: + vfree(array.entries); return r; } -static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i) -{ - struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i]; - struct kvm_cpuid_entry2 *ej; - int j = i; - int nent = vcpu->arch.cpuid_nent; - - e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT; - /* when no next entry is found, the current entry[i] is reselected */ - do { - j = (j + 1) % nent; - ej = &vcpu->arch.cpuid_entries[j]; - } while (ej->function != e->function); - - ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; - - return j; -} - -/* find an entry with matching function, matching index (if needed), and that - * should be read next (if it's stateful) */ -static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e, - u32 function, u32 index) -{ - if (e->function != function) - return 0; - if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index) - return 0; - if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) && - !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT)) - return 0; - return 1; -} - struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu, u32 function, u32 index) { + struct kvm_cpuid_entry2 *e; int i; - struct kvm_cpuid_entry2 *best = NULL; for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { - struct kvm_cpuid_entry2 *e; - e = &vcpu->arch.cpuid_entries[i]; - if (is_matching_cpuid_entry(e, function, index)) { - if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) - move_to_next_stateful_cpuid_entry(vcpu, i); - best = e; - break; - } + + if (e->function == function && (e->index == index || + !(e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX))) + return e; } - return best; + return NULL; } EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry); /* - * If the basic or extended CPUID leaf requested is higher than the - * maximum supported basic or extended leaf, respectively, then it is - * out of range. + * Intel CPUID semantics treats any query for an out-of-range leaf as if the + * highest basic leaf (i.e. CPUID.0H:EAX) were requested. AMD CPUID semantics + * returns all zeroes for any undefined leaf, whether or not the leaf is in + * range. Centaur/VIA follows Intel semantics. + * + * A leaf is considered out-of-range if its function is higher than the maximum + * supported leaf of its associated class or if its associated class does not + * exist. + * + * There are three primary classes to be considered, with their respective + * ranges described as " - [, - ] inclusive. A primary + * class exists if a guest CPUID entry for its leaf exists. For a given + * class, CPUID..EAX contains the max supported leaf for the class. + * + * - Basic: 0x00000000 - 0x3fffffff, 0x50000000 - 0x7fffffff + * - Hypervisor: 0x40000000 - 0x4fffffff + * - Extended: 0x80000000 - 0xbfffffff + * - Centaur: 0xc0000000 - 0xcfffffff + * + * The Hypervisor class is further subdivided into sub-classes that each act as + * their own indepdent class associated with a 0x100 byte range. E.g. if Qemu + * is advertising support for both HyperV and KVM, the resulting Hypervisor + * CPUID sub-classes are: + * + * - HyperV: 0x40000000 - 0x400000ff + * - KVM: 0x40000100 - 0x400001ff */ -static bool cpuid_function_in_range(struct kvm_vcpu *vcpu, u32 function) +static struct kvm_cpuid_entry2 * +get_out_of_range_cpuid_entry(struct kvm_vcpu *vcpu, u32 *fn_ptr, u32 index) { - struct kvm_cpuid_entry2 *max; + struct kvm_cpuid_entry2 *basic, *class; + u32 function = *fn_ptr; - max = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0); - return max && function <= max->eax; + basic = kvm_find_cpuid_entry(vcpu, 0, 0); + if (!basic) + return NULL; + + if (is_guest_vendor_amd(basic->ebx, basic->ecx, basic->edx) || + is_guest_vendor_hygon(basic->ebx, basic->ecx, basic->edx)) + return NULL; + + if (function >= 0x40000000 && function <= 0x4fffffff) + class = kvm_find_cpuid_entry(vcpu, function & 0xffffff00, 0); + else if (function >= 0xc0000000) + class = kvm_find_cpuid_entry(vcpu, 0xc0000000, 0); + else + class = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0); + + if (class && function <= class->eax) + return NULL; + + /* + * Leaf specific adjustments are also applied when redirecting to the + * max basic entry, e.g. if the max basic leaf is 0xb but there is no + * entry for CPUID.0xb.index (see below), then the output value for EDX + * needs to be pulled from CPUID.0xb.1. + */ + *fn_ptr = basic->eax; + + /* + * The class does not exist or the requested function is out of range; + * the effective CPUID entry is the max basic leaf. Note, the index of + * the original requested leaf is observed! + */ + return kvm_find_cpuid_entry(vcpu, basic->eax, index); } bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, - u32 *ecx, u32 *edx, bool check_limit) + u32 *ecx, u32 *edx, bool exact_only) { - u32 function = *eax, index = *ecx; + u32 orig_function = *eax, function = *eax, index = *ecx; struct kvm_cpuid_entry2 *entry; - struct kvm_cpuid_entry2 *max; - bool found; + bool exact, used_max_basic = false; entry = kvm_find_cpuid_entry(vcpu, function, index); - found = entry; - /* - * Intel CPUID semantics treats any query for an out-of-range - * leaf as if the highest basic leaf (i.e. CPUID.0H:EAX) were - * requested. AMD CPUID semantics returns all zeroes for any - * undefined leaf, whether or not the leaf is in range. - */ - if (!entry && check_limit && !guest_cpuid_is_amd(vcpu) && - !cpuid_function_in_range(vcpu, function)) { - max = kvm_find_cpuid_entry(vcpu, 0, 0); - if (max) { - function = max->eax; - entry = kvm_find_cpuid_entry(vcpu, function, index); - } + exact = !!entry; + + if (!entry && !exact_only) { + entry = get_out_of_range_cpuid_entry(vcpu, &function, index); + used_max_basic = !!entry; } + if (entry) { *eax = entry->eax; *ebx = entry->ebx; @@ -1049,8 +1028,9 @@ bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, } } } - trace_kvm_cpuid(function, *eax, *ebx, *ecx, *edx, found); - return found; + trace_kvm_cpuid(orig_function, index, *eax, *ebx, *ecx, *edx, exact, + used_max_basic); + return exact; } EXPORT_SYMBOL_GPL(kvm_cpuid); @@ -1063,7 +1043,7 @@ int kvm_emulate_cpuid(struct kvm_vcpu *vcpu) eax = kvm_rax_read(vcpu); ecx = kvm_rcx_read(vcpu); - kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, true); + kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false); kvm_rax_write(vcpu, eax); kvm_rbx_write(vcpu, ebx); kvm_rcx_write(vcpu, ecx); diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 7366c618aa04..63a70f6a3df3 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -6,8 +6,10 @@ #include #include +extern u32 kvm_cpu_caps[NCAPINTS] __read_mostly; +void kvm_set_cpu_caps(void); + int kvm_update_cpuid(struct kvm_vcpu *vcpu); -bool kvm_mpx_supported(void); struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu, u32 function, u32 index); int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid, @@ -23,7 +25,7 @@ int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, struct kvm_cpuid_entry2 __user *entries); bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, - u32 *ecx, u32 *edx, bool check_limit); + u32 *ecx, u32 *edx, bool exact_only); int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); @@ -64,7 +66,7 @@ static const struct cpuid_reg reverse_cpuid[] = { * and can't be used by KVM to query/control guest capabilities. And obviously * the leaf being queried must have an entry in the lookup table. */ -static __always_inline void reverse_cpuid_check(unsigned x86_leaf) +static __always_inline void reverse_cpuid_check(unsigned int x86_leaf) { BUILD_BUG_ON(x86_leaf == CPUID_LNX_1); BUILD_BUG_ON(x86_leaf == CPUID_LNX_2); @@ -88,24 +90,18 @@ static __always_inline u32 __feature_bit(int x86_feature) #define feature_bit(name) __feature_bit(X86_FEATURE_##name) -static __always_inline struct cpuid_reg x86_feature_cpuid(unsigned x86_feature) +static __always_inline struct cpuid_reg x86_feature_cpuid(unsigned int x86_feature) { - unsigned x86_leaf = x86_feature / 32; + unsigned int x86_leaf = x86_feature / 32; reverse_cpuid_check(x86_leaf); return reverse_cpuid[x86_leaf]; } -static __always_inline int *guest_cpuid_get_register(struct kvm_vcpu *vcpu, unsigned x86_feature) +static __always_inline u32 *__cpuid_entry_get_reg(struct kvm_cpuid_entry2 *entry, + u32 reg) { - struct kvm_cpuid_entry2 *entry; - const struct cpuid_reg cpuid = x86_feature_cpuid(x86_feature); - - entry = kvm_find_cpuid_entry(vcpu, cpuid.function, cpuid.index); - if (!entry) - return NULL; - - switch (cpuid.reg) { + switch (reg) { case CPUID_EAX: return &entry->eax; case CPUID_EBX: @@ -120,9 +116,86 @@ static __always_inline int *guest_cpuid_get_register(struct kvm_vcpu *vcpu, unsi } } -static __always_inline bool guest_cpuid_has(struct kvm_vcpu *vcpu, unsigned x86_feature) +static __always_inline u32 *cpuid_entry_get_reg(struct kvm_cpuid_entry2 *entry, + unsigned int x86_feature) { - int *reg; + const struct cpuid_reg cpuid = x86_feature_cpuid(x86_feature); + + return __cpuid_entry_get_reg(entry, cpuid.reg); +} + +static __always_inline u32 cpuid_entry_get(struct kvm_cpuid_entry2 *entry, + unsigned int x86_feature) +{ + u32 *reg = cpuid_entry_get_reg(entry, x86_feature); + + return *reg & __feature_bit(x86_feature); +} + +static __always_inline bool cpuid_entry_has(struct kvm_cpuid_entry2 *entry, + unsigned int x86_feature) +{ + return cpuid_entry_get(entry, x86_feature); +} + +static __always_inline void cpuid_entry_clear(struct kvm_cpuid_entry2 *entry, + unsigned int x86_feature) +{ + u32 *reg = cpuid_entry_get_reg(entry, x86_feature); + + *reg &= ~__feature_bit(x86_feature); +} + +static __always_inline void cpuid_entry_set(struct kvm_cpuid_entry2 *entry, + unsigned int x86_feature) +{ + u32 *reg = cpuid_entry_get_reg(entry, x86_feature); + + *reg |= __feature_bit(x86_feature); +} + +static __always_inline void cpuid_entry_change(struct kvm_cpuid_entry2 *entry, + unsigned int x86_feature, + bool set) +{ + u32 *reg = cpuid_entry_get_reg(entry, x86_feature); + + /* + * Open coded instead of using cpuid_entry_{clear,set}() to coerce the + * compiler into using CMOV instead of Jcc when possible. + */ + if (set) + *reg |= __feature_bit(x86_feature); + else + *reg &= ~__feature_bit(x86_feature); +} + +static __always_inline void cpuid_entry_override(struct kvm_cpuid_entry2 *entry, + enum cpuid_leafs leaf) +{ + u32 *reg = cpuid_entry_get_reg(entry, leaf * 32); + + BUILD_BUG_ON(leaf >= ARRAY_SIZE(kvm_cpu_caps)); + *reg = kvm_cpu_caps[leaf]; +} + +static __always_inline u32 *guest_cpuid_get_register(struct kvm_vcpu *vcpu, + unsigned int x86_feature) +{ + const struct cpuid_reg cpuid = x86_feature_cpuid(x86_feature); + struct kvm_cpuid_entry2 *entry; + + entry = kvm_find_cpuid_entry(vcpu, cpuid.function, cpuid.index); + if (!entry) + return NULL; + + return __cpuid_entry_get_reg(entry, cpuid.reg); +} + +static __always_inline bool guest_cpuid_has(struct kvm_vcpu *vcpu, + unsigned int x86_feature) +{ + u32 *reg; reg = guest_cpuid_get_register(vcpu, x86_feature); if (!reg) @@ -131,21 +204,24 @@ static __always_inline bool guest_cpuid_has(struct kvm_vcpu *vcpu, unsigned x86_ return *reg & __feature_bit(x86_feature); } -static __always_inline void guest_cpuid_clear(struct kvm_vcpu *vcpu, unsigned x86_feature) +static __always_inline void guest_cpuid_clear(struct kvm_vcpu *vcpu, + unsigned int x86_feature) { - int *reg; + u32 *reg; reg = guest_cpuid_get_register(vcpu, x86_feature); if (reg) *reg &= ~__feature_bit(x86_feature); } -static inline bool guest_cpuid_is_amd(struct kvm_vcpu *vcpu) +static inline bool guest_cpuid_is_amd_or_hygon(struct kvm_vcpu *vcpu) { struct kvm_cpuid_entry2 *best; best = kvm_find_cpuid_entry(vcpu, 0, 0); - return best && best->ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx; + return best && + (is_guest_vendor_amd(best->ebx, best->ecx, best->edx) || + is_guest_vendor_hygon(best->ebx, best->ecx, best->edx)); } static inline int guest_cpuid_family(struct kvm_vcpu *vcpu) @@ -192,4 +268,39 @@ static inline bool cpuid_fault_enabled(struct kvm_vcpu *vcpu) MSR_MISC_FEATURES_ENABLES_CPUID_FAULT; } +static __always_inline void kvm_cpu_cap_clear(unsigned int x86_feature) +{ + unsigned int x86_leaf = x86_feature / 32; + + reverse_cpuid_check(x86_leaf); + kvm_cpu_caps[x86_leaf] &= ~__feature_bit(x86_feature); +} + +static __always_inline void kvm_cpu_cap_set(unsigned int x86_feature) +{ + unsigned int x86_leaf = x86_feature / 32; + + reverse_cpuid_check(x86_leaf); + kvm_cpu_caps[x86_leaf] |= __feature_bit(x86_feature); +} + +static __always_inline u32 kvm_cpu_cap_get(unsigned int x86_feature) +{ + unsigned int x86_leaf = x86_feature / 32; + + reverse_cpuid_check(x86_leaf); + return kvm_cpu_caps[x86_leaf] & __feature_bit(x86_feature); +} + +static __always_inline bool kvm_cpu_cap_has(unsigned int x86_feature) +{ + return !!kvm_cpu_cap_get(x86_feature); +} + +static __always_inline void kvm_cpu_cap_check_and_set(unsigned int x86_feature) +{ + if (boot_cpu_has(x86_feature)) + kvm_cpu_cap_set(x86_feature); +} + #endif diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index bc00642e5d3b..bddaba9c68dd 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -20,7 +20,7 @@ #include #include "kvm_cache_regs.h" -#include +#include "kvm_emulate.h" #include #include #include @@ -665,6 +665,17 @@ static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector, ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg); } +static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt) +{ + return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48; +} + +static inline bool emul_is_noncanonical_address(u64 la, + struct x86_emulate_ctxt *ctxt) +{ + return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la; +} + /* * x86 defines three classes of vector instructions: explicitly * aligned, explicitly unaligned, and the rest, which change behaviour @@ -2711,10 +2722,8 @@ static bool vendor_intel(struct x86_emulate_ctxt *ctxt) u32 eax, ebx, ecx, edx; eax = ecx = 0; - ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false); - return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx - && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx - && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx; + ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true); + return is_guest_vendor_intel(ebx, ecx, edx); } static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt) @@ -2731,36 +2740,18 @@ static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt) eax = 0x00000000; ecx = 0x00000000; - ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false); + ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true); /* - * Intel ("GenuineIntel") - * remark: Intel CPUs only support "syscall" in 64bit - * longmode. Also an 64bit guest with a - * 32bit compat-app running will #UD !! While this - * behaviour can be fixed (by emulating) into AMD - * response - CPUs of AMD can't behave like Intel. + * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a + * 64bit guest with a 32bit compat-app running will #UD !! While this + * behaviour can be fixed (by emulating) into AMD response - CPUs of + * AMD can't behave like Intel. */ - if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx && - ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx && - edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx) + if (is_guest_vendor_intel(ebx, ecx, edx)) return false; - /* AMD ("AuthenticAMD") */ - if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx && - ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx && - edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx) - return true; - - /* AMD ("AMDisbetter!") */ - if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx && - ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx && - edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx) - return true; - - /* Hygon ("HygonGenuine") */ - if (ebx == X86EMUL_CPUID_VENDOR_HygonGenuine_ebx && - ecx == X86EMUL_CPUID_VENDOR_HygonGenuine_ecx && - edx == X86EMUL_CPUID_VENDOR_HygonGenuine_edx) + if (is_guest_vendor_amd(ebx, ecx, edx) || + is_guest_vendor_hygon(ebx, ecx, edx)) return true; /* @@ -3980,7 +3971,7 @@ static int em_cpuid(struct x86_emulate_ctxt *ctxt) eax = reg_read(ctxt, VCPU_REGS_RAX); ecx = reg_read(ctxt, VCPU_REGS_RCX); - ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true); + ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false); *reg_write(ctxt, VCPU_REGS_RAX) = eax; *reg_write(ctxt, VCPU_REGS_RBX) = ebx; *reg_write(ctxt, VCPU_REGS_RCX) = ecx; @@ -4250,7 +4241,7 @@ static int check_cr_write(struct x86_emulate_ctxt *ctxt) eax = 0x80000008; ecx = 0; if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, - &edx, false)) + &edx, true)) maxphyaddr = eax & 0xff; else maxphyaddr = 36; diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index a86fda7a1d03..bcefa9d4e57e 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1022,7 +1022,7 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, addr = gfn_to_hva(kvm, gfn); if (kvm_is_error_hva(addr)) return 1; - kvm_x86_ops->patch_hypercall(vcpu, instructions); + kvm_x86_ops.patch_hypercall(vcpu, instructions); ((unsigned char *)instructions)[3] = 0xc3; /* ret */ if (__copy_to_user((void __user *)addr, instructions, 4)) return 1; @@ -1607,7 +1607,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) * hypercall generates UD from non zero cpl and real mode * per HYPER-V spec */ - if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) { + if (kvm_x86_ops.get_cpl(vcpu) != 0 || !is_protmode(vcpu)) { kvm_queue_exception(vcpu, UD_VECTOR); return 1; } @@ -1800,8 +1800,8 @@ int kvm_vcpu_ioctl_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, }; int i, nent = ARRAY_SIZE(cpuid_entries); - if (kvm_x86_ops->nested_get_evmcs_version) - evmcs_ver = kvm_x86_ops->nested_get_evmcs_version(vcpu); + if (kvm_x86_ops.nested_get_evmcs_version) + evmcs_ver = kvm_x86_ops.nested_get_evmcs_version(vcpu); /* Skip NESTED_FEATURES if eVMCS is not supported */ if (!evmcs_ver) diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index b24c606ac04b..febca334c320 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c @@ -367,7 +367,7 @@ static void pit_load_count(struct kvm_pit *pit, int channel, u32 val) { struct kvm_kpit_state *ps = &pit->pit_state; - pr_debug("load_count val is %d, channel is %d\n", val, channel); + pr_debug("load_count val is %u, channel is %d\n", val, channel); /* * The largest possible initial count is 0; this is equivalent diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h index 58767020de41..62558b9bdda7 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/kvm_cache_regs.h @@ -68,7 +68,7 @@ static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg) return 0; if (!kvm_register_is_available(vcpu, reg)) - kvm_x86_ops->cache_reg(vcpu, reg); + kvm_x86_ops.cache_reg(vcpu, reg); return vcpu->arch.regs[reg]; } @@ -108,7 +108,7 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index) might_sleep(); /* on svm */ if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR)) - kvm_x86_ops->cache_reg(vcpu, VCPU_EXREG_PDPTR); + kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_PDPTR); return vcpu->arch.walk_mmu->pdptrs[index]; } @@ -117,7 +117,7 @@ static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask) { ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS; if (tmask & vcpu->arch.cr0_guest_owned_bits) - kvm_x86_ops->decache_cr0_guest_bits(vcpu); + kvm_x86_ops.decache_cr0_guest_bits(vcpu); return vcpu->arch.cr0 & mask; } @@ -130,14 +130,14 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask) { ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS; if (tmask & vcpu->arch.cr4_guest_owned_bits) - kvm_x86_ops->decache_cr4_guest_bits(vcpu); + kvm_x86_ops.decache_cr4_guest_bits(vcpu); return vcpu->arch.cr4 & mask; } static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu) { if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3)) - kvm_x86_ops->cache_reg(vcpu, VCPU_EXREG_CR3); + kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR3); return vcpu->arch.cr3; } diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h similarity index 93% rename from arch/x86/include/asm/kvm_emulate.h rename to arch/x86/kvm/kvm_emulate.h index c06e8353efd3..43c93ffa76ed 100644 --- a/arch/x86/include/asm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -221,7 +221,7 @@ struct x86_emulate_ops { enum x86_intercept_stage stage); bool (*get_cpuid)(struct x86_emulate_ctxt *ctxt, u32 *eax, u32 *ebx, - u32 *ecx, u32 *edx, bool check_limit); + u32 *ecx, u32 *edx, bool exact_only); bool (*guest_has_long_mode)(struct x86_emulate_ctxt *ctxt); bool (*guest_has_movbe)(struct x86_emulate_ctxt *ctxt); bool (*guest_has_fxsr)(struct x86_emulate_ctxt *ctxt); @@ -301,6 +301,7 @@ struct fastop; typedef void (*fastop_t)(struct fastop *); struct x86_emulate_ctxt { + void *vcpu; const struct x86_emulate_ops *ops; /* Register state before/after emulation. */ @@ -319,6 +320,10 @@ struct x86_emulate_ctxt { bool have_exception; struct x86_exception exception; + /* GPA available */ + bool gpa_available; + gpa_t gpa_val; + /* * decode cache */ @@ -329,9 +334,6 @@ struct x86_emulate_ctxt { u8 intercept; u8 op_bytes; u8 ad_bytes; - struct operand src; - struct operand src2; - struct operand dst; union { int (*execute)(struct x86_emulate_ctxt *ctxt); fastop_t fop; @@ -359,6 +361,11 @@ struct x86_emulate_ctxt { u8 seg_override; u64 d; unsigned long _eip; + + /* Here begins the usercopy section. */ + struct operand src; + struct operand src2; + struct operand dst; struct operand memop; unsigned long _regs[NR_VCPU_REGS]; struct operand *memopp; @@ -388,6 +395,34 @@ struct x86_emulate_ctxt { #define X86EMUL_CPUID_VENDOR_GenuineIntel_ecx 0x6c65746e #define X86EMUL_CPUID_VENDOR_GenuineIntel_edx 0x49656e69 +#define X86EMUL_CPUID_VENDOR_CentaurHauls_ebx 0x746e6543 +#define X86EMUL_CPUID_VENDOR_CentaurHauls_ecx 0x736c7561 +#define X86EMUL_CPUID_VENDOR_CentaurHauls_edx 0x48727561 + +static inline bool is_guest_vendor_intel(u32 ebx, u32 ecx, u32 edx) +{ + return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx && + ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx && + edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx; +} + +static inline bool is_guest_vendor_amd(u32 ebx, u32 ecx, u32 edx) +{ + return (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx && + ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx && + edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx) || + (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx && + ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx && + edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx); +} + +static inline bool is_guest_vendor_hygon(u32 ebx, u32 ecx, u32 edx) +{ + return ebx == X86EMUL_CPUID_VENDOR_HygonGenuine_ebx && + ecx == X86EMUL_CPUID_VENDOR_HygonGenuine_ecx && + edx == X86EMUL_CPUID_VENDOR_HygonGenuine_edx; +} + enum x86_intercept_stage { X86_ICTP_NONE = 0, /* Allow zero-init to not match anything */ X86_ICPT_PRE_EXCEPT, diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 7356a56e6282..ca80daf8f878 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -164,14 +164,28 @@ static void kvm_apic_map_free(struct rcu_head *rcu) kvfree(map); } -static void recalculate_apic_map(struct kvm *kvm) +void kvm_recalculate_apic_map(struct kvm *kvm) { struct kvm_apic_map *new, *old = NULL; struct kvm_vcpu *vcpu; int i; u32 max_id = 255; /* enough space for any xAPIC ID */ + if (!kvm->arch.apic_map_dirty) { + /* + * Read kvm->arch.apic_map_dirty before + * kvm->arch.apic_map + */ + smp_rmb(); + return; + } + mutex_lock(&kvm->arch.apic_map_lock); + if (!kvm->arch.apic_map_dirty) { + /* Someone else has updated the map. */ + mutex_unlock(&kvm->arch.apic_map_lock); + return; + } kvm_for_each_vcpu(i, vcpu, kvm) if (kvm_apic_present(vcpu)) @@ -236,6 +250,12 @@ static void recalculate_apic_map(struct kvm *kvm) old = rcu_dereference_protected(kvm->arch.apic_map, lockdep_is_held(&kvm->arch.apic_map_lock)); rcu_assign_pointer(kvm->arch.apic_map, new); + /* + * Write kvm->arch.apic_map before + * clearing apic->apic_map_dirty + */ + smp_wmb(); + kvm->arch.apic_map_dirty = false; mutex_unlock(&kvm->arch.apic_map_lock); if (old) @@ -257,20 +277,20 @@ static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) else static_key_slow_inc(&apic_sw_disabled.key); - recalculate_apic_map(apic->vcpu->kvm); + apic->vcpu->kvm->arch.apic_map_dirty = true; } } static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) { kvm_lapic_set_reg(apic, APIC_ID, id << 24); - recalculate_apic_map(apic->vcpu->kvm); + apic->vcpu->kvm->arch.apic_map_dirty = true; } static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) { kvm_lapic_set_reg(apic, APIC_LDR, id); - recalculate_apic_map(apic->vcpu->kvm); + apic->vcpu->kvm->arch.apic_map_dirty = true; } static inline u32 kvm_apic_calc_x2apic_ldr(u32 id) @@ -286,7 +306,7 @@ static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) kvm_lapic_set_reg(apic, APIC_ID, id); kvm_lapic_set_reg(apic, APIC_LDR, ldr); - recalculate_apic_map(apic->vcpu->kvm); + apic->vcpu->kvm->arch.apic_map_dirty = true; } static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) @@ -294,11 +314,6 @@ static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); } -static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) -{ - return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; -} - static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) { return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; @@ -448,7 +463,7 @@ static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) if (unlikely(vcpu->arch.apicv_active)) { /* need to update RVI */ kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); - kvm_x86_ops->hwapic_irr_update(vcpu, + kvm_x86_ops.hwapic_irr_update(vcpu, apic_find_highest_irr(apic)); } else { apic->irr_pending = false; @@ -473,7 +488,7 @@ static inline void apic_set_isr(int vec, struct kvm_lapic *apic) * just set SVI. */ if (unlikely(vcpu->arch.apicv_active)) - kvm_x86_ops->hwapic_isr_update(vcpu, vec); + kvm_x86_ops.hwapic_isr_update(vcpu, vec); else { ++apic->isr_count; BUG_ON(apic->isr_count > MAX_APIC_VECTOR); @@ -521,7 +536,7 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) * and must be left alone. */ if (unlikely(vcpu->arch.apicv_active)) - kvm_x86_ops->hwapic_isr_update(vcpu, + kvm_x86_ops.hwapic_isr_update(vcpu, apic_find_highest_isr(apic)); else { --apic->isr_count; @@ -659,7 +674,7 @@ static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) { int highest_irr; if (apic->vcpu->arch.apicv_active) - highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu); + highest_irr = kvm_x86_ops.sync_pir_to_irr(apic->vcpu); else highest_irr = apic_find_highest_irr(apic); if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) @@ -1048,7 +1063,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, apic->regs + APIC_TMR); } - if (kvm_x86_ops->deliver_posted_interrupt(vcpu, vector)) { + if (kvm_x86_ops.deliver_posted_interrupt(vcpu, vector)) { kvm_lapic_set_irr(vector, apic); kvm_make_request(KVM_REQ_EVENT, vcpu); kvm_vcpu_kick(vcpu); @@ -1226,7 +1241,7 @@ void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) } EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); -static void apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high) +void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high) { struct kvm_lapic_irq irq; @@ -1737,7 +1752,7 @@ static void cancel_hv_timer(struct kvm_lapic *apic) { WARN_ON(preemptible()); WARN_ON(!apic->lapic_timer.hv_timer_in_use); - kvm_x86_ops->cancel_hv_timer(apic->vcpu); + kvm_x86_ops.cancel_hv_timer(apic->vcpu); apic->lapic_timer.hv_timer_in_use = false; } @@ -1748,13 +1763,13 @@ static bool start_hv_timer(struct kvm_lapic *apic) bool expired; WARN_ON(preemptible()); - if (!kvm_x86_ops->set_hv_timer) + if (!kvm_x86_ops.set_hv_timer) return false; if (!ktimer->tscdeadline) return false; - if (kvm_x86_ops->set_hv_timer(vcpu, ktimer->tscdeadline, &expired)) + if (kvm_x86_ops.set_hv_timer(vcpu, ktimer->tscdeadline, &expired)) return false; ktimer->hv_timer_in_use = true; @@ -1917,7 +1932,7 @@ int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) case APIC_DFR: if (!apic_x2apic_mode(apic)) { kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); - recalculate_apic_map(apic->vcpu->kvm); + apic->vcpu->kvm->arch.apic_map_dirty = true; } else ret = 1; break; @@ -1946,7 +1961,7 @@ int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) case APIC_ICR: /* No delay here, so we always clear the pending bit */ val &= ~(1 << 12); - apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2)); + kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2)); kvm_lapic_set_reg(apic, APIC_ICR, val); break; @@ -2023,6 +2038,8 @@ int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) break; } + kvm_recalculate_apic_map(apic->vcpu->kvm); + return ret; } EXPORT_SYMBOL_GPL(kvm_lapic_reg_write); @@ -2171,7 +2188,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) static_key_slow_dec_deferred(&apic_hw_disabled); } else { static_key_slow_inc(&apic_hw_disabled.key); - recalculate_apic_map(vcpu->kvm); + vcpu->kvm->arch.apic_map_dirty = true; } } @@ -2179,7 +2196,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) - kvm_x86_ops->set_virtual_apic_mode(vcpu); + kvm_x86_ops.set_virtual_apic_mode(vcpu); apic->base_address = apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_BASE; @@ -2212,6 +2229,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) if (!apic) return; + vcpu->kvm->arch.apic_map_dirty = false; /* Stop the timer in case it's a reset to an active apic */ hrtimer_cancel(&apic->lapic_timer.timer); @@ -2256,13 +2274,15 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) vcpu->arch.pv_eoi.msr_val = 0; apic_update_ppr(apic); if (vcpu->arch.apicv_active) { - kvm_x86_ops->apicv_post_state_restore(vcpu); - kvm_x86_ops->hwapic_irr_update(vcpu, -1); - kvm_x86_ops->hwapic_isr_update(vcpu, -1); + kvm_x86_ops.apicv_post_state_restore(vcpu); + kvm_x86_ops.hwapic_irr_update(vcpu, -1); + kvm_x86_ops.hwapic_isr_update(vcpu, -1); } vcpu->arch.apic_arb_prio = 0; vcpu->arch.apic_attention = 0; + + kvm_recalculate_apic_map(vcpu->kvm); } /* @@ -2484,17 +2504,18 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) struct kvm_lapic *apic = vcpu->arch.apic; int r; - kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); /* set SPIV separately to get count of SW disabled APICs right */ apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); r = kvm_apic_state_fixup(vcpu, s, true); - if (r) + if (r) { + kvm_recalculate_apic_map(vcpu->kvm); return r; + } memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s)); - recalculate_apic_map(vcpu->kvm); + kvm_recalculate_apic_map(vcpu->kvm); kvm_apic_set_version(vcpu); apic_update_ppr(apic); @@ -2506,10 +2527,10 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) kvm_apic_update_apicv(vcpu); apic->highest_isr_cache = -1; if (vcpu->arch.apicv_active) { - kvm_x86_ops->apicv_post_state_restore(vcpu); - kvm_x86_ops->hwapic_irr_update(vcpu, + kvm_x86_ops.apicv_post_state_restore(vcpu); + kvm_x86_ops.hwapic_irr_update(vcpu, apic_find_highest_irr(apic)); - kvm_x86_ops->hwapic_isr_update(vcpu, + kvm_x86_ops.hwapic_isr_update(vcpu, apic_find_highest_isr(apic)); } kvm_make_request(KVM_REQ_EVENT, vcpu); diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index ec6fbfe325cf..40ed6ed22751 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -78,6 +78,7 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu); void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu); +void kvm_recalculate_apic_map(struct kvm *kvm); void kvm_apic_set_version(struct kvm_vcpu *vcpu); int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val); int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, @@ -95,6 +96,7 @@ void kvm_apic_update_apicv(struct kvm_vcpu *vcpu); bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map); +void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high); u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info); diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index a647601c9e1c..8a3b1bce722a 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -95,11 +95,11 @@ static inline unsigned long kvm_get_active_pcid(struct kvm_vcpu *vcpu) return kvm_get_pcid(vcpu, kvm_read_cr3(vcpu)); } -static inline void kvm_mmu_load_cr3(struct kvm_vcpu *vcpu) +static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu) { if (VALID_PAGE(vcpu->arch.mmu->root_hpa)) - vcpu->arch.mmu->set_cr3(vcpu, vcpu->arch.mmu->root_hpa | - kvm_get_active_pcid(vcpu)); + kvm_x86_ops.load_mmu_pgd(vcpu, vcpu->arch.mmu->root_hpa | + kvm_get_active_pcid(vcpu)); } int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, @@ -170,8 +170,8 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned pte_access, unsigned pte_pkey, unsigned pfec) { - int cpl = kvm_x86_ops->get_cpl(vcpu); - unsigned long rflags = kvm_x86_ops->get_rflags(vcpu); + int cpl = kvm_x86_ops.get_cpl(vcpu); + unsigned long rflags = kvm_x86_ops.get_rflags(vcpu); /* * If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1. diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 87e9ba27ada1..8071952e9cf2 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -19,6 +19,7 @@ #include "mmu.h" #include "x86.h" #include "kvm_cache_regs.h" +#include "kvm_emulate.h" #include "cpuid.h" #include @@ -86,6 +87,8 @@ __MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint"); */ bool tdp_enabled = false; +static int max_page_level __read_mostly; + enum { AUDIT_PRE_PAGE_FAULT, AUDIT_POST_PAGE_FAULT, @@ -215,17 +218,6 @@ struct kvm_shadow_walk_iterator { unsigned index; }; -static const union kvm_mmu_page_role mmu_base_role_mask = { - .cr0_wp = 1, - .gpte_is_8_bytes = 1, - .nxe = 1, - .smep_andnot_wp = 1, - .smap_andnot_wp = 1, - .smm = 1, - .guest_mode = 1, - .ad_disabled = 1, -}; - #define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker) \ for (shadow_walk_init_using_root(&(_walker), (_vcpu), \ (_root), (_addr)); \ @@ -313,7 +305,7 @@ kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu); static inline bool kvm_available_flush_tlb_with_range(void) { - return kvm_x86_ops->tlb_remote_flush_with_range; + return kvm_x86_ops.tlb_remote_flush_with_range; } static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm, @@ -321,8 +313,8 @@ static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm, { int ret = -ENOTSUPP; - if (range && kvm_x86_ops->tlb_remote_flush_with_range) - ret = kvm_x86_ops->tlb_remote_flush_with_range(kvm, range); + if (range && kvm_x86_ops.tlb_remote_flush_with_range) + ret = kvm_x86_ops.tlb_remote_flush_with_range(kvm, range); if (ret) kvm_flush_remote_tlbs(kvm); @@ -1650,7 +1642,7 @@ static bool spte_set_dirty(u64 *sptep) rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep); /* - * Similar to the !kvm_x86_ops->slot_disable_log_dirty case, + * Similar to the !kvm_x86_ops.slot_disable_log_dirty case, * do not bother adding back write access to pages marked * SPTE_AD_WRPROT_ONLY_MASK. */ @@ -1739,8 +1731,8 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask) { - if (kvm_x86_ops->enable_log_dirty_pt_masked) - kvm_x86_ops->enable_log_dirty_pt_masked(kvm, slot, gfn_offset, + if (kvm_x86_ops.enable_log_dirty_pt_masked) + kvm_x86_ops.enable_log_dirty_pt_masked(kvm, slot, gfn_offset, mask); else kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); @@ -1755,8 +1747,8 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, */ int kvm_arch_write_log_dirty(struct kvm_vcpu *vcpu) { - if (kvm_x86_ops->write_log_dirty) - return kvm_x86_ops->write_log_dirty(vcpu); + if (kvm_x86_ops.write_log_dirty) + return kvm_x86_ops.write_log_dirty(vcpu); return 0; } @@ -3044,7 +3036,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, if (level > PT_PAGE_TABLE_LEVEL) spte |= PT_PAGE_SIZE_MASK; if (tdp_enabled) - spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn, + spte |= kvm_x86_ops.get_mt_mask(vcpu, gfn, kvm_is_mmio_pfn(pfn)); if (host_writable) @@ -3292,7 +3284,7 @@ static int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, if (!slot) return PT_PAGE_TABLE_LEVEL; - max_level = min(max_level, kvm_x86_ops->get_lpage_level()); + max_level = min(max_level, max_page_level); for ( ; max_level > PT_PAGE_TABLE_LEVEL; max_level--) { linfo = lpage_info_slot(gfn, slot, max_level); if (!linfo->disallow_lpage) @@ -3568,8 +3560,7 @@ static bool fast_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, * write-protected for dirty-logging or access tracking. */ if ((error_code & PFERR_WRITE_MASK) && - spte_can_locklessly_be_made_writable(spte)) - { + spte_can_locklessly_be_made_writable(spte)) { new_spte |= PT_WRITABLE_MASK; /* @@ -3731,7 +3722,9 @@ static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root); } else BUG(); - vcpu->arch.mmu->root_cr3 = vcpu->arch.mmu->get_cr3(vcpu); + + /* root_cr3 is ignored for direct MMUs. */ + vcpu->arch.mmu->root_cr3 = 0; return 0; } @@ -3743,7 +3736,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) gfn_t root_gfn, root_cr3; int i; - root_cr3 = vcpu->arch.mmu->get_cr3(vcpu); + root_cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu); root_gfn = root_cr3 >> PAGE_SHIFT; if (mmu_check_root(vcpu, root_gfn)) @@ -4080,7 +4073,7 @@ static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id; arch.gfn = gfn; arch.direct_map = vcpu->arch.mmu->direct_map; - arch.cr3 = vcpu->arch.mmu->get_cr3(vcpu); + arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu); return kvm_setup_async_pf(vcpu, cr2_or_gpa, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch); @@ -4252,6 +4245,14 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu, context->nx = false; } +static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t cr3, + union kvm_mmu_page_role role) +{ + return (role.direct || cr3 == root->cr3) && + VALID_PAGE(root->hpa) && page_header(root->hpa) && + role.word == page_header(root->hpa)->role.word; +} + /* * Find out if a previously cached root matching the new CR3/role is available. * The current root is also inserted into the cache. @@ -4270,12 +4271,13 @@ static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_cr3, root.cr3 = mmu->root_cr3; root.hpa = mmu->root_hpa; + if (is_root_usable(&root, new_cr3, new_role)) + return true; + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { swap(root, mmu->prev_roots[i]); - if (new_cr3 == root.cr3 && VALID_PAGE(root.hpa) && - page_header(root.hpa) != NULL && - new_role.word == page_header(root.hpa)->role.word) + if (is_root_usable(&root, new_cr3, new_role)) break; } @@ -4309,7 +4311,7 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, * accompanied by KVM_REQ_MMU_RELOAD, which will free * the root set here and allocate a new one. */ - kvm_make_request(KVM_REQ_LOAD_CR3, vcpu); + kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); if (!skip_tlb_flush) { kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); @@ -4508,7 +4510,8 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, cpuid_maxphyaddr(vcpu), context->root_level, context->nx, guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES), - is_pse(vcpu), guest_cpuid_is_amd(vcpu)); + is_pse(vcpu), + guest_cpuid_is_amd_or_hygon(vcpu)); } static void @@ -4874,7 +4877,6 @@ static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu) ext.cr4_smap = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMAP); ext.cr4_pse = !!is_pse(vcpu); ext.cr4_pke = !!kvm_read_cr4_bits(vcpu, X86_CR4_PKE); - ext.cr4_la57 = !!kvm_read_cr4_bits(vcpu, X86_CR4_LA57); ext.maxphyaddr = cpuid_maxphyaddr(vcpu); ext.valid = 1; @@ -4907,7 +4909,7 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); role.base.ad_disabled = (shadow_accessed_mask == 0); - role.base.level = kvm_x86_ops->get_tdp_level(vcpu); + role.base.level = kvm_x86_ops.get_tdp_level(vcpu); role.base.direct = true; role.base.gpte_is_8_bytes = true; @@ -4920,7 +4922,6 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) union kvm_mmu_role new_role = kvm_calc_tdp_mmu_root_page_role(vcpu, false); - new_role.base.word &= mmu_base_role_mask.word; if (new_role.as_u64 == context->mmu_role.as_u64) return; @@ -4929,10 +4930,9 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) context->sync_page = nonpaging_sync_page; context->invlpg = nonpaging_invlpg; context->update_pte = nonpaging_update_pte; - context->shadow_root_level = kvm_x86_ops->get_tdp_level(vcpu); + context->shadow_root_level = kvm_x86_ops.get_tdp_level(vcpu); context->direct_map = true; - context->set_cr3 = kvm_x86_ops->set_tdp_cr3; - context->get_cr3 = get_cr3; + context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; context->inject_page_fault = kvm_inject_page_fault; @@ -4992,7 +4992,6 @@ void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu) union kvm_mmu_role new_role = kvm_calc_shadow_mmu_root_page_role(vcpu, false); - new_role.base.word &= mmu_base_role_mask.word; if (new_role.as_u64 == context->mmu_role.as_u64) return; @@ -5012,14 +5011,14 @@ EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu); static union kvm_mmu_role kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, - bool execonly) + bool execonly, u8 level) { union kvm_mmu_role role = {0}; /* SMM flag is inherited from root_mmu */ role.base.smm = vcpu->arch.root_mmu.mmu_role.base.smm; - role.base.level = PT64_ROOT_4LEVEL; + role.base.level = level; role.base.gpte_is_8_bytes = true; role.base.direct = false; role.base.ad_disabled = !accessed_dirty; @@ -5043,17 +5042,17 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, bool accessed_dirty, gpa_t new_eptp) { struct kvm_mmu *context = vcpu->arch.mmu; + u8 level = vmx_eptp_page_walk_level(new_eptp); union kvm_mmu_role new_role = kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, - execonly); + execonly, level); __kvm_mmu_new_cr3(vcpu, new_eptp, new_role.base, false); - new_role.base.word &= mmu_base_role_mask.word; if (new_role.as_u64 == context->mmu_role.as_u64) return; - context->shadow_root_level = PT64_ROOT_4LEVEL; + context->shadow_root_level = level; context->nx = true; context->ept_ad = accessed_dirty; @@ -5062,7 +5061,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, context->sync_page = ept_sync_page; context->invlpg = ept_invlpg; context->update_pte = ept_update_pte; - context->root_level = PT64_ROOT_4LEVEL; + context->root_level = level; context->direct_map = false; context->mmu_role.as_u64 = new_role.as_u64; @@ -5079,8 +5078,7 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu) struct kvm_mmu *context = vcpu->arch.mmu; kvm_init_shadow_mmu(vcpu); - context->set_cr3 = kvm_x86_ops->set_cr3; - context->get_cr3 = get_cr3; + context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; context->inject_page_fault = kvm_inject_page_fault; } @@ -5090,12 +5088,11 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) union kvm_mmu_role new_role = kvm_calc_mmu_role_common(vcpu, false); struct kvm_mmu *g_context = &vcpu->arch.nested_mmu; - new_role.base.word &= mmu_base_role_mask.word; if (new_role.as_u64 == g_context->mmu_role.as_u64) return; g_context->mmu_role.as_u64 = new_role.as_u64; - g_context->get_cr3 = get_cr3; + g_context->get_guest_pgd = get_cr3; g_context->get_pdptr = kvm_pdptr_read; g_context->inject_page_fault = kvm_inject_page_fault; @@ -5185,8 +5182,8 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) kvm_mmu_sync_roots(vcpu); if (r) goto out; - kvm_mmu_load_cr3(vcpu); - kvm_x86_ops->tlb_flush(vcpu, true); + kvm_mmu_load_pgd(vcpu); + kvm_x86_ops.tlb_flush(vcpu, true); out: return r; } @@ -5329,6 +5326,22 @@ static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte) return spte; } +/* + * Ignore various flags when determining if a SPTE can be immediately + * overwritten for the current MMU. + * - level: explicitly checked in mmu_pte_write_new_pte(), and will never + * match the current MMU role, as MMU's level tracks the root level. + * - access: updated based on the new guest PTE + * - quadrant: handled by get_written_sptes() + * - invalid: always false (loop only walks valid shadow pages) + */ +static const union kvm_mmu_page_role role_ign = { + .level = 0xf, + .access = 0x7, + .quadrant = 0x3, + .invalid = 0x1, +}; + static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, int bytes, struct kvm_page_track_notifier_node *node) @@ -5384,8 +5397,8 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, entry = *spte; mmu_page_zap_pte(vcpu->kvm, sp, spte); if (gentry && - !((sp->role.word ^ base_role) - & mmu_base_role_mask.word) && rmap_can_add(vcpu)) + !((sp->role.word ^ base_role) & ~role_ign.word) && + rmap_can_add(vcpu)) mmu_pte_write_new_pte(vcpu, sp, spte, &gentry); if (need_remote_flush(entry, *spte)) remote_flush = true; @@ -5416,18 +5429,12 @@ EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt); int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, void *insn, int insn_len) { - int r, emulation_type = 0; + int r, emulation_type = EMULTYPE_PF; bool direct = vcpu->arch.mmu->direct_map; if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) return RET_PF_RETRY; - /* With shadow page tables, fault_address contains a GVA or nGPA. */ - if (vcpu->arch.mmu->direct_map) { - vcpu->arch.gpa_available = true; - vcpu->arch.gpa_val = cr2_or_gpa; - } - r = RET_PF_INVALID; if (unlikely(error_code & PFERR_RSVD_MASK)) { r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct); @@ -5471,7 +5478,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, * for L1 isn't going to magically fix whatever issue cause L2 to fail. */ if (!mmio_info_in_cache(vcpu, cr2_or_gpa, direct) && !is_guest_mode(vcpu)) - emulation_type = EMULTYPE_ALLOW_RETRY; + emulation_type |= EMULTYPE_ALLOW_RETRY_PF; emulate: /* * On AMD platforms, under certain conditions insn_len may be zero on #NPF. @@ -5481,7 +5488,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, * guest, with the exception of AMD Erratum 1096 which is unrecoverable. */ if (unlikely(insn && !insn_len)) { - if (!kvm_x86_ops->need_emulation_on_page_fault(vcpu)) + if (!kvm_x86_ops.need_emulation_on_page_fault(vcpu)) return 1; } @@ -5516,7 +5523,7 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) if (VALID_PAGE(mmu->prev_roots[i].hpa)) mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); - kvm_x86_ops->tlb_flush_gva(vcpu, gva); + kvm_x86_ops.tlb_flush_gva(vcpu, gva); ++vcpu->stat.invlpg; } EXPORT_SYMBOL_GPL(kvm_mmu_invlpg); @@ -5541,7 +5548,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) } if (tlb_flush) - kvm_x86_ops->tlb_flush_gva(vcpu, gva); + kvm_x86_ops.tlb_flush_gva(vcpu, gva); ++vcpu->stat.invlpg; @@ -5553,18 +5560,25 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) } EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva); -void kvm_enable_tdp(void) +void kvm_configure_mmu(bool enable_tdp, int tdp_page_level) { - tdp_enabled = true; -} -EXPORT_SYMBOL_GPL(kvm_enable_tdp); + tdp_enabled = enable_tdp; -void kvm_disable_tdp(void) -{ - tdp_enabled = false; + /* + * max_page_level reflects the capabilities of KVM's MMU irrespective + * of kernel support, e.g. KVM may be capable of using 1GB pages when + * the kernel is not. But, KVM never creates a page size greater than + * what is used by the kernel for any given HVA, i.e. the kernel's + * capabilities are ultimately consulted by kvm_mmu_hugepage_adjust(). + */ + if (tdp_enabled) + max_page_level = tdp_page_level; + else if (boot_cpu_has(X86_FEATURE_GBPAGES)) + max_page_level = PT_PDPE_LEVEL; + else + max_page_level = PT_DIRECTORY_LEVEL; } -EXPORT_SYMBOL_GPL(kvm_disable_tdp); - +EXPORT_SYMBOL_GPL(kvm_configure_mmu); /* The return value indicates if tlb flush on all vcpus is needed. */ typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head); @@ -5658,7 +5672,7 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) * SVM's 32-bit NPT support, TDP paging doesn't use PAE paging and can * skip allocating the PDP table. */ - if (tdp_enabled && kvm_x86_ops->get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) + if (tdp_enabled && kvm_x86_ops.get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) return 0; page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); @@ -5860,22 +5874,16 @@ static bool slot_rmap_write_protect(struct kvm *kvm, } void kvm_mmu_slot_remove_write_access(struct kvm *kvm, - struct kvm_memory_slot *memslot) + struct kvm_memory_slot *memslot, + int start_level) { bool flush; spin_lock(&kvm->mmu_lock); - flush = slot_handle_all_level(kvm, memslot, slot_rmap_write_protect, - false); + flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect, + start_level, PT_MAX_HUGEPAGE_LEVEL, false); spin_unlock(&kvm->mmu_lock); - /* - * kvm_mmu_slot_remove_write_access() and kvm_vm_ioctl_get_dirty_log() - * which do tlb flush out of mmu-lock should be serialized by - * kvm->slots_lock otherwise tlb flush would be missed. - */ - lockdep_assert_held(&kvm->slots_lock); - /* * We can flush all the TLBs out of the mmu lock without TLB * corruption since we just change the spte from writable to @@ -5888,8 +5896,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, * on PT_WRITABLE_MASK anymore. */ if (flush) - kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn, - memslot->npages); + kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); } static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, @@ -5941,6 +5948,21 @@ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, spin_unlock(&kvm->mmu_lock); } +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ + /* + * All current use cases for flushing the TLBs for a specific memslot + * are related to dirty logging, and do the TLB flush out of mmu_lock. + * The interaction between the various operations on memslot must be + * serialized by slots_locks to ensure the TLB flush from one operation + * is observed by any other operation on the same memslot. + */ + lockdep_assert_held(&kvm->slots_lock); + kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn, + memslot->npages); +} + void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, struct kvm_memory_slot *memslot) { @@ -5950,8 +5972,6 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false); spin_unlock(&kvm->mmu_lock); - lockdep_assert_held(&kvm->slots_lock); - /* * It's also safe to flush TLBs out of mmu lock here as currently this * function is only used for dirty logging, in which case flushing TLB @@ -5959,8 +5979,7 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, * dirty_bitmap. */ if (flush) - kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn, - memslot->npages); + kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); } EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty); @@ -5974,12 +5993,8 @@ void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, false); spin_unlock(&kvm->mmu_lock); - /* see kvm_mmu_slot_remove_write_access */ - lockdep_assert_held(&kvm->slots_lock); - if (flush) - kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn, - memslot->npages); + kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); } EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access); @@ -5992,12 +6007,8 @@ void kvm_mmu_slot_set_dirty(struct kvm *kvm, flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false); spin_unlock(&kvm->mmu_lock); - lockdep_assert_held(&kvm->slots_lock); - - /* see kvm_mmu_slot_leaf_clear_dirty */ if (flush) - kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn, - memslot->npages); + kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); } EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty); diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 3521e2d176f2..ddc1ec3bdacd 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -14,22 +14,18 @@ #include #include -#include #include #include "mmu.h" -void kvm_page_track_free_memslot(struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) +void kvm_page_track_free_memslot(struct kvm_memory_slot *slot) { int i; - for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) - if (!dont || free->arch.gfn_track[i] != - dont->arch.gfn_track[i]) { - kvfree(free->arch.gfn_track[i]); - free->arch.gfn_track[i] = NULL; - } + for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { + kvfree(slot->arch.gfn_track[i]); + slot->arch.gfn_track[i] = NULL; + } } int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, @@ -48,7 +44,7 @@ int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, return 0; track_free: - kvm_page_track_free_memslot(slot, NULL); + kvm_page_track_free_memslot(slot); return -ENOMEM; } diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 21a3320f166a..9bdf9b7d9a96 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -66,7 +66,7 @@ #define PT_GUEST_ACCESSED_SHIFT 8 #define PT_HAVE_ACCESSED_DIRTY(mmu) ((mmu)->ept_ad) #define CMPXCHG cmpxchg64 - #define PT_MAX_FULL_LEVELS 4 + #define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL #else #error Invalid PTTYPE value #endif @@ -333,7 +333,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, trace_kvm_mmu_pagetable_walk(addr, access); retry_walk: walker->level = mmu->root_level; - pte = mmu->get_cr3(vcpu); + pte = mmu->get_guest_pgd(vcpu); have_ad = PT_HAVE_ACCESSED_DIRTY(mmu); #if PTTYPE == 64 diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index bcc6a73d6628..a5078841bdac 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -111,7 +111,7 @@ static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, .config = config, }; - attr.sample_period = (-pmc->counter) & pmc_bitmask(pmc); + attr.sample_period = get_sample_period(pmc, pmc->counter); if (in_tx) attr.config |= HSW_IN_TX; @@ -158,7 +158,7 @@ static bool pmc_resume_counter(struct kvm_pmc *pmc) /* recalibrate sample period and check if it's accepted by perf core */ if (perf_event_period(pmc->perf_event, - (-pmc->counter) & pmc_bitmask(pmc))) + get_sample_period(pmc, pmc->counter))) return false; /* reuse perf_event to serve as pmc_reprogram_counter() does*/ @@ -211,7 +211,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) ARCH_PERFMON_EVENTSEL_CMASK | HSW_IN_TX | HSW_IN_TX_CHECKPOINTED))) { - config = kvm_x86_ops->pmu_ops->find_arch_event(pmc_to_pmu(pmc), + config = kvm_x86_ops.pmu_ops->find_arch_event(pmc_to_pmu(pmc), event_select, unit_mask); if (config != PERF_COUNT_HW_MAX) @@ -265,7 +265,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) pmc->current_config = (u64)ctrl; pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE, - kvm_x86_ops->pmu_ops->find_fixed_event(idx), + kvm_x86_ops.pmu_ops->find_fixed_event(idx), !(en_field & 0x2), /* exclude user */ !(en_field & 0x1), /* exclude kernel */ pmi, false, false); @@ -274,7 +274,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter); void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx) { - struct kvm_pmc *pmc = kvm_x86_ops->pmu_ops->pmc_idx_to_pmc(pmu, pmc_idx); + struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, pmc_idx); if (!pmc) return; @@ -296,7 +296,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) int bit; for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) { - struct kvm_pmc *pmc = kvm_x86_ops->pmu_ops->pmc_idx_to_pmc(pmu, bit); + struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, bit); if (unlikely(!pmc || !pmc->perf_event)) { clear_bit(bit, pmu->reprogram_pmi); @@ -318,7 +318,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) /* check if idx is a valid index to access PMU */ int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { - return kvm_x86_ops->pmu_ops->is_valid_rdpmc_ecx(vcpu, idx); + return kvm_x86_ops.pmu_ops->is_valid_rdpmc_ecx(vcpu, idx); } bool is_vmware_backdoor_pmc(u32 pmc_idx) @@ -368,7 +368,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) if (is_vmware_backdoor_pmc(idx)) return kvm_pmu_rdpmc_vmware(vcpu, idx, data); - pmc = kvm_x86_ops->pmu_ops->rdpmc_ecx_to_pmc(vcpu, idx, &mask); + pmc = kvm_x86_ops.pmu_ops->rdpmc_ecx_to_pmc(vcpu, idx, &mask); if (!pmc) return 1; @@ -384,14 +384,14 @@ void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { - return kvm_x86_ops->pmu_ops->msr_idx_to_pmc(vcpu, msr) || - kvm_x86_ops->pmu_ops->is_valid_msr(vcpu, msr); + return kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr) || + kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, msr); } static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); - struct kvm_pmc *pmc = kvm_x86_ops->pmu_ops->msr_idx_to_pmc(vcpu, msr); + struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr); if (pmc) __set_bit(pmc->idx, pmu->pmc_in_use); @@ -399,13 +399,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *data) { - return kvm_x86_ops->pmu_ops->get_msr(vcpu, msr, data); + return kvm_x86_ops.pmu_ops->get_msr(vcpu, msr, data); } int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); - return kvm_x86_ops->pmu_ops->set_msr(vcpu, msr_info); + return kvm_x86_ops.pmu_ops->set_msr(vcpu, msr_info); } /* refresh PMU settings. This function generally is called when underlying @@ -414,7 +414,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) */ void kvm_pmu_refresh(struct kvm_vcpu *vcpu) { - kvm_x86_ops->pmu_ops->refresh(vcpu); + kvm_x86_ops.pmu_ops->refresh(vcpu); } void kvm_pmu_reset(struct kvm_vcpu *vcpu) @@ -422,7 +422,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); irq_work_sync(&pmu->irq_work); - kvm_x86_ops->pmu_ops->reset(vcpu); + kvm_x86_ops.pmu_ops->reset(vcpu); } void kvm_pmu_init(struct kvm_vcpu *vcpu) @@ -430,7 +430,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); memset(pmu, 0, sizeof(*pmu)); - kvm_x86_ops->pmu_ops->init(vcpu); + kvm_x86_ops.pmu_ops->init(vcpu); init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn); pmu->event_count = 0; pmu->need_cleanup = false; @@ -462,7 +462,7 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) pmu->pmc_in_use, X86_PMC_IDX_MAX); for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) { - pmc = kvm_x86_ops->pmu_ops->pmc_idx_to_pmc(pmu, i); + pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, i); if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc)) pmc_stop_counter(pmc); diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 13332984b6d5..a6c78a797cb1 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -88,7 +88,7 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc) static inline bool pmc_is_enabled(struct kvm_pmc *pmc) { - return kvm_x86_ops->pmu_ops->pmc_is_enabled(pmc); + return kvm_x86_ops.pmu_ops->pmc_is_enabled(pmc); } static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu, @@ -129,6 +129,15 @@ static inline struct kvm_pmc *get_fixed_pmc(struct kvm_pmu *pmu, u32 msr) return NULL; } +static inline u64 get_sample_period(struct kvm_pmc *pmc, u64 counter_value) +{ + u64 sample_period = (-counter_value) & pmc_bitmask(pmc); + + if (!sample_period) + sample_period = pmc_bitmask(pmc) + 1; + return sample_period; +} + void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel); void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int fixed_idx); void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx); diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index 216364cb65a3..851e9cc79930 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -522,10 +522,31 @@ static void recalc_intercepts(struct vcpu_svm *svm) h = &svm->nested.hsave->control; g = &svm->nested; - c->intercept_cr = h->intercept_cr | g->intercept_cr; - c->intercept_dr = h->intercept_dr | g->intercept_dr; - c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions; - c->intercept = h->intercept | g->intercept; + c->intercept_cr = h->intercept_cr; + c->intercept_dr = h->intercept_dr; + c->intercept_exceptions = h->intercept_exceptions; + c->intercept = h->intercept; + + if (svm->vcpu.arch.hflags & HF_VINTR_MASK) { + /* We only want the cr8 intercept bits of L1 */ + c->intercept_cr &= ~(1U << INTERCEPT_CR8_READ); + c->intercept_cr &= ~(1U << INTERCEPT_CR8_WRITE); + + /* + * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not + * affect any interrupt we may want to inject; therefore, + * interrupt window vmexits are irrelevant to L0. + */ + c->intercept &= ~(1ULL << INTERCEPT_VINTR); + } + + /* We don't want to see VMMCALLs from a nested guest */ + c->intercept &= ~(1ULL << INTERCEPT_VMMCALL); + + c->intercept_cr |= g->intercept_cr; + c->intercept_dr |= g->intercept_dr; + c->intercept_exceptions |= g->intercept_exceptions; + c->intercept |= g->intercept; } static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm) @@ -630,6 +651,11 @@ static inline void clr_intercept(struct vcpu_svm *svm, int bit) recalc_intercepts(svm); } +static inline bool is_intercept(struct vcpu_svm *svm, int bit) +{ + return (svm->vmcb->control.intercept & (1ULL << bit)) != 0; +} + static inline bool vgif_enabled(struct vcpu_svm *svm) { return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK); @@ -1209,6 +1235,7 @@ static int avic_ga_log_notifier(u32 ga_tag) u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); + trace_kvm_avic_ga_log(vm_id, vcpu_id); spin_lock_irqsave(&svm_vm_data_hash_lock, flags); hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { @@ -1370,6 +1397,29 @@ static void svm_hardware_teardown(void) iopm_base = 0; } +static __init void svm_set_cpu_caps(void) +{ + kvm_set_cpu_caps(); + + supported_xss = 0; + + /* CPUID 0x80000001 and 0x8000000A (SVM features) */ + if (nested) { + kvm_cpu_cap_set(X86_FEATURE_SVM); + + if (nrips) + kvm_cpu_cap_set(X86_FEATURE_NRIPS); + + if (npt_enabled) + kvm_cpu_cap_set(X86_FEATURE_NPT); + } + + /* CPUID 0x80000008 */ + if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || + boot_cpu_has(X86_FEATURE_AMD_SSBD)) + kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); +} + static __init int svm_hardware_setup(void) { int cpu; @@ -1388,6 +1438,8 @@ static __init int svm_hardware_setup(void) init_msrpm_offsets(); + supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); + if (boot_cpu_has(X86_FEATURE_NX)) kvm_enable_efer_bits(EFER_NX); @@ -1435,16 +1487,11 @@ static __init int svm_hardware_setup(void) if (!boot_cpu_has(X86_FEATURE_NPT)) npt_enabled = false; - if (npt_enabled && !npt) { - printk(KERN_INFO "kvm: Nested Paging disabled\n"); + if (npt_enabled && !npt) npt_enabled = false; - } - if (npt_enabled) { - printk(KERN_INFO "kvm: Nested Paging enabled\n"); - kvm_enable_tdp(); - } else - kvm_disable_tdp(); + kvm_configure_mmu(npt_enabled, PT_PDPE_LEVEL); + pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); if (nrips) { if (!boot_cpu_has(X86_FEATURE_NRIPS)) @@ -1480,6 +1527,8 @@ static __init int svm_hardware_setup(void) pr_info("Virtual GIF supported\n"); } + svm_set_cpu_caps(); + return 0; err: @@ -1939,19 +1988,6 @@ static void __unregister_enc_region_locked(struct kvm *kvm, kfree(region); } -static struct kvm *svm_vm_alloc(void) -{ - struct kvm_svm *kvm_svm = __vmalloc(sizeof(struct kvm_svm), - GFP_KERNEL_ACCOUNT | __GFP_ZERO, - PAGE_KERNEL); - return &kvm_svm->kvm; -} - -static void svm_vm_free(struct kvm *kvm) -{ - vfree(to_kvm_svm(kvm)); -} - static void sev_vm_destroy(struct kvm *kvm) { struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; @@ -2186,7 +2222,7 @@ static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) } init_vmcb(svm); - kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true); + kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, false); kvm_rdx_write(vcpu, eax); if (kvm_vcpu_apicv_active(vcpu) && !init_event) @@ -2420,14 +2456,38 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) } } +static inline void svm_enable_vintr(struct vcpu_svm *svm) +{ + struct vmcb_control_area *control; + + /* The following fields are ignored when AVIC is enabled */ + WARN_ON(kvm_vcpu_apicv_active(&svm->vcpu)); + + /* + * This is just a dummy VINTR to actually cause a vmexit to happen. + * Actual injection of virtual interrupts happens through EVENTINJ. + */ + control = &svm->vmcb->control; + control->int_vector = 0x0; + control->int_ctl &= ~V_INTR_PRIO_MASK; + control->int_ctl |= V_IRQ_MASK | + ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); + mark_dirty(svm->vmcb, VMCB_INTR); +} + static void svm_set_vintr(struct vcpu_svm *svm) { set_intercept(svm, INTERCEPT_VINTR); + if (is_intercept(svm, INTERCEPT_VINTR)) + svm_enable_vintr(svm); } static void svm_clear_vintr(struct vcpu_svm *svm) { clr_intercept(svm, INTERCEPT_VINTR); + + svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; + mark_dirty(svm->vmcb, VMCB_INTR); } static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) @@ -2983,15 +3043,6 @@ static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) return pdpte; } -static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu, - unsigned long root) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm->vmcb->control.nested_cr3 = __sme_set(root); - mark_dirty(svm->vmcb, VMCB_NPT); -} - static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, struct x86_exception *fault) { @@ -3027,8 +3078,7 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) vcpu->arch.mmu = &vcpu->arch.guest_mmu; kvm_init_shadow_mmu(vcpu); - vcpu->arch.mmu->set_cr3 = nested_svm_set_tdp_cr3; - vcpu->arch.mmu->get_cr3 = nested_svm_get_tdp_cr3; + vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; vcpu->arch.mmu->shadow_root_level = get_npt_level(vcpu); @@ -3089,43 +3139,36 @@ static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, return vmexit; } -/* This function returns true if it is save to enable the irq window */ -static inline bool nested_svm_intr(struct vcpu_svm *svm) +static void nested_svm_intr(struct vcpu_svm *svm) { - if (!is_guest_mode(&svm->vcpu)) - return true; - - if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK)) - return true; - - if (!(svm->vcpu.arch.hflags & HF_HIF_MASK)) - return false; - - /* - * if vmexit was already requested (by intercepted exception - * for instance) do not overwrite it with "external interrupt" - * vmexit. - */ - if (svm->nested.exit_required) - return false; - svm->vmcb->control.exit_code = SVM_EXIT_INTR; svm->vmcb->control.exit_info_1 = 0; svm->vmcb->control.exit_info_2 = 0; - if (svm->nested.intercept & 1ULL) { - /* - * The #vmexit can't be emulated here directly because this - * code path runs with irqs and preemption disabled. A - * #vmexit emulation might sleep. Only signal request for - * the #vmexit here. - */ - svm->nested.exit_required = true; - trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); - return false; + /* nested_svm_vmexit this gets called afterwards from handle_exit */ + svm->nested.exit_required = true; + trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); +} + +static bool nested_exit_on_intr(struct vcpu_svm *svm) +{ + return (svm->nested.intercept & 1ULL); +} + +static int svm_check_nested_events(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + bool block_nested_events = + kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required; + + if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(svm)) { + if (block_nested_events) + return -EBUSY; + nested_svm_intr(svm); + return 0; } - return true; + return 0; } /* This function returns true if it is save to enable the nmi window */ @@ -3244,9 +3287,6 @@ static int nested_svm_exit_special(struct vcpu_svm *svm) return NESTED_EXIT_CONTINUE; } -/* - * If this function returns true, this #vmexit was already handled - */ static int nested_svm_intercept(struct vcpu_svm *svm) { u32 exit_code = svm->vmcb->control.exit_code; @@ -3521,6 +3561,9 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) static bool nested_vmcb_checks(struct vmcb *vmcb) { + if ((vmcb->save.efer & EFER_SVME) == 0) + return false; + if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0) return false; @@ -3537,6 +3580,10 @@ static bool nested_vmcb_checks(struct vmcb *vmcb) static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, struct vmcb *nested_vmcb, struct kvm_host_map *map) { + bool evaluate_pending_interrupts = + is_intercept(svm, INTERCEPT_VINTR) || + is_intercept(svm, INTERCEPT_IRET); + if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF) svm->vcpu.arch.hflags |= HF_HIF_MASK; else @@ -3596,15 +3643,6 @@ static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, else svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; - if (svm->vcpu.arch.hflags & HF_VINTR_MASK) { - /* We only want the cr8 intercept bits of the guest */ - clr_cr_intercept(svm, INTERCEPT_CR8_READ); - clr_cr_intercept(svm, INTERCEPT_CR8_WRITE); - } - - /* We don't want to see VMMCALLs from a nested guest */ - clr_intercept(svm, INTERCEPT_VMMCALL); - svm->vcpu.arch.tsc_offset += nested_vmcb->control.tsc_offset; svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; @@ -3632,7 +3670,21 @@ static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, svm->nested.vmcb = vmcb_gpa; + /* + * If L1 had a pending IRQ/NMI before executing VMRUN, + * which wasn't delivered because it was disallowed (e.g. + * interrupts disabled), L0 needs to evaluate if this pending + * event should cause an exit from L2 to L1 or be delivered + * directly to L2. + * + * Usually this would be handled by the processor noticing an + * IRQ/NMI window request. However, VMRUN can unblock interrupts + * by implicitly setting GIF, so force L0 to perform pending event + * evaluation by requesting a KVM_REQ_EVENT. + */ enable_gif(svm); + if (unlikely(evaluate_pending_interrupts)) + kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); mark_all_dirty(svm->vmcb); } @@ -3834,11 +3886,8 @@ static int clgi_interception(struct vcpu_svm *svm) disable_gif(svm); /* After a CLGI no interrupts should come */ - if (!kvm_vcpu_apicv_active(&svm->vcpu)) { + if (!kvm_vcpu_apicv_active(&svm->vcpu)) svm_clear_vintr(svm); - svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; - mark_dirty(svm->vmcb, VMCB_INTR); - } return ret; } @@ -5124,19 +5173,6 @@ static void svm_inject_nmi(struct kvm_vcpu *vcpu) ++vcpu->stat.nmi_injections; } -static inline void svm_inject_irq(struct vcpu_svm *svm, int irq) -{ - struct vmcb_control_area *control; - - /* The following fields are ignored when AVIC is enabled */ - control = &svm->vmcb->control; - control->int_vector = irq; - control->int_ctl &= ~V_INTR_PRIO_MASK; - control->int_ctl |= V_IRQ_MASK | - ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); - mark_dirty(svm->vmcb, VMCB_INTR); -} - static void svm_set_irq(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -5525,18 +5561,15 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; - int ret; if (!gif_set(svm) || (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)) return 0; - ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF); - - if (is_guest_mode(vcpu)) - return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK); - - return ret; + if (is_guest_mode(vcpu) && (svm->vcpu.arch.hflags & HF_VINTR_MASK)) + return !!(svm->vcpu.arch.hflags & HF_HIF_MASK); + else + return !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF); } static void enable_irq_window(struct kvm_vcpu *vcpu) @@ -5551,7 +5584,7 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) * enabled, the STGI interception will not occur. Enable the irq * window under the assumption that the hardware will set the GIF. */ - if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) { + if (vgif_enabled(svm) || gif_set(svm)) { /* * IRQ window is not needed when AVIC is enabled, * unless we have pending ExtINT since it cannot be injected @@ -5560,7 +5593,6 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) */ svm_toggle_avic_for_irq_window(vcpu, false); svm_set_vintr(svm); - svm_inject_irq(svm, 0x0); } } @@ -5946,24 +5978,30 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) } STACK_FRAME_NON_STANDARD(svm_vcpu_run); -static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) +static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long root) { struct vcpu_svm *svm = to_svm(vcpu); + bool update_guest_cr3 = true; + unsigned long cr3; - svm->vmcb->save.cr3 = __sme_set(root); - mark_dirty(svm->vmcb, VMCB_CR); -} + cr3 = __sme_set(root); + if (npt_enabled) { + svm->vmcb->control.nested_cr3 = cr3; + mark_dirty(svm->vmcb, VMCB_NPT); -static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root) -{ - struct vcpu_svm *svm = to_svm(vcpu); + /* Loading L2's CR3 is handled by enter_svm_guest_mode. */ + if (is_guest_mode(vcpu)) + update_guest_cr3 = false; + else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) + cr3 = vcpu->arch.cr3; + else /* CR3 is already up-to-date. */ + update_guest_cr3 = false; + } - svm->vmcb->control.nested_cr3 = __sme_set(root); - mark_dirty(svm->vmcb, VMCB_NPT); - - /* Also sync guest cr3 here in case we live migrate */ - svm->vmcb->save.cr3 = kvm_read_cr3(vcpu); - mark_dirty(svm->vmcb, VMCB_CR); + if (update_guest_cr3) { + svm->vmcb->save.cr3 = cr3; + mark_dirty(svm->vmcb, VMCB_CR); + } } static int is_disabled(void) @@ -6025,12 +6063,19 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu) boot_cpu_has(X86_FEATURE_XSAVES); /* Update nrips enabled cache */ - svm->nrips_enabled = !!guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS); + svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) && + guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS); if (!kvm_vcpu_apicv_active(vcpu)) return; - guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC); + /* + * AVIC does not work with an x2APIC mode guest. If the X2APIC feature + * is exposed to the guest, disable AVIC. + */ + if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC)) + kvm_request_apicv_update(vcpu->kvm, false, + APICV_INHIBIT_REASON_X2APIC); /* * Currently, AVIC does not work with nested virtualization. @@ -6041,88 +6086,11 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu) APICV_INHIBIT_REASON_NESTED); } -#define F feature_bit - -static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) -{ - switch (func) { - case 0x1: - if (avic) - entry->ecx &= ~F(X2APIC); - break; - case 0x80000001: - if (nested) - entry->ecx |= (1 << 2); /* Set SVM bit */ - break; - case 0x80000008: - if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || - boot_cpu_has(X86_FEATURE_AMD_SSBD)) - entry->ebx |= F(VIRT_SSBD); - break; - case 0x8000000A: - entry->eax = 1; /* SVM revision 1 */ - entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper - ASID emulation to nested SVM */ - entry->ecx = 0; /* Reserved */ - entry->edx = 0; /* Per default do not support any - additional features */ - - /* Support next_rip if host supports it */ - if (boot_cpu_has(X86_FEATURE_NRIPS)) - entry->edx |= F(NRIPS); - - /* Support NPT for the guest if enabled */ - if (npt_enabled) - entry->edx |= F(NPT); - - } -} - -static int svm_get_lpage_level(void) -{ - return PT_PDPE_LEVEL; -} - -static bool svm_rdtscp_supported(void) -{ - return boot_cpu_has(X86_FEATURE_RDTSCP); -} - -static bool svm_invpcid_supported(void) -{ - return false; -} - -static bool svm_mpx_supported(void) -{ - return false; -} - -static bool svm_xsaves_supported(void) -{ - return boot_cpu_has(X86_FEATURE_XSAVES); -} - -static bool svm_umip_emulated(void) -{ - return false; -} - -static bool svm_pt_supported(void) -{ - return false; -} - static bool svm_has_wbinvd_exit(void) { return true; } -static bool svm_pku_supported(void) -{ - return false; -} - #define PRE_EX(exit) { .exit_code = (exit), \ .stage = X86_ICPT_PRE_EXCEPT, } #define POST_EX(exit) { .exit_code = (exit), \ @@ -6189,7 +6157,8 @@ static const struct __x86_intercept { static int svm_check_intercept(struct kvm_vcpu *vcpu, struct x86_instruction_info *info, - enum x86_intercept_stage stage) + enum x86_intercept_stage stage, + struct x86_exception *exception) { struct vcpu_svm *svm = to_svm(vcpu); int vmexit, ret = X86EMUL_CONTINUE; @@ -7371,7 +7340,7 @@ static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) * TODO: Last condition latch INIT signals on vCPU when * vCPU is in guest-mode and vmcb12 defines intercept on INIT. * To properly emulate the INIT intercept, SVM should implement - * kvm_x86_ops->check_nested_events() and call nested_svm_vmexit() + * kvm_x86_ops.check_nested_events() and call nested_svm_vmexit() * there if an INIT signal is pending. */ return !gif_set(svm) || @@ -7384,7 +7353,8 @@ static bool svm_check_apicv_inhibit_reasons(ulong bit) BIT(APICV_INHIBIT_REASON_HYPERV) | BIT(APICV_INHIBIT_REASON_NESTED) | BIT(APICV_INHIBIT_REASON_IRQWIN) | - BIT(APICV_INHIBIT_REASON_PIT_REINJ); + BIT(APICV_INHIBIT_REASON_PIT_REINJ) | + BIT(APICV_INHIBIT_REASON_X2APIC); return supported & BIT(bit); } @@ -7394,12 +7364,8 @@ static void svm_pre_update_apicv_exec_ctrl(struct kvm *kvm, bool activate) avic_update_access_page(kvm, activate); } -static struct kvm_x86_ops svm_x86_ops __ro_after_init = { - .cpu_has_kvm_support = has_svm, - .disabled_by_bios = is_disabled, - .hardware_setup = svm_hardware_setup, +static struct kvm_x86_ops svm_x86_ops __initdata = { .hardware_unsetup = svm_hardware_teardown, - .check_processor_compatibility = svm_check_processor_compat, .hardware_enable = svm_hardware_enable, .hardware_disable = svm_hardware_disable, .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr, @@ -7409,8 +7375,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = { .vcpu_free = svm_free_vcpu, .vcpu_reset = svm_vcpu_reset, - .vm_alloc = svm_vm_alloc, - .vm_free = svm_vm_free, + .vm_size = sizeof(struct kvm_svm), .vm_init = svm_vm_init, .vm_destroy = svm_vm_destroy, @@ -7432,7 +7397,6 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = { .decache_cr0_guest_bits = svm_decache_cr0_guest_bits, .decache_cr4_guest_bits = svm_decache_cr4_guest_bits, .set_cr0 = svm_set_cr0, - .set_cr3 = svm_set_cr3, .set_cr4 = svm_set_cr4, .set_efer = svm_set_efer, .get_idt = svm_get_idt, @@ -7485,26 +7449,14 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = { .get_exit_info = svm_get_exit_info, - .get_lpage_level = svm_get_lpage_level, - .cpuid_update = svm_cpuid_update, - .rdtscp_supported = svm_rdtscp_supported, - .invpcid_supported = svm_invpcid_supported, - .mpx_supported = svm_mpx_supported, - .xsaves_supported = svm_xsaves_supported, - .umip_emulated = svm_umip_emulated, - .pt_supported = svm_pt_supported, - .pku_supported = svm_pku_supported, - - .set_supported_cpuid = svm_set_supported_cpuid, - .has_wbinvd_exit = svm_has_wbinvd_exit, .read_l1_tsc_offset = svm_read_l1_tsc_offset, .write_l1_tsc_offset = svm_write_l1_tsc_offset, - .set_tdp_cr3 = set_tdp_cr3, + .load_mmu_pgd = svm_load_mmu_pgd, .check_intercept = svm_check_intercept, .handle_exit_irqoff = svm_handle_exit_irqoff, @@ -7534,11 +7486,22 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = { .need_emulation_on_page_fault = svm_need_emulation_on_page_fault, .apic_init_signal_blocked = svm_apic_init_signal_blocked, + + .check_nested_events = svm_check_nested_events, +}; + +static struct kvm_x86_init_ops svm_init_ops __initdata = { + .cpu_has_kvm_support = has_svm, + .disabled_by_bios = is_disabled, + .hardware_setup = svm_hardware_setup, + .check_processor_compatibility = svm_check_processor_compat, + + .runtime_ops = &svm_x86_ops, }; static int __init svm_init(void) { - return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm), + return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm), THIS_MODULE); } diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index cef5a344fedb..249062f24b94 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -151,32 +151,38 @@ TRACE_EVENT(kvm_fast_mmio, * Tracepoint for cpuid. */ TRACE_EVENT(kvm_cpuid, - TP_PROTO(unsigned int function, unsigned long rax, unsigned long rbx, - unsigned long rcx, unsigned long rdx, bool found), - TP_ARGS(function, rax, rbx, rcx, rdx, found), + TP_PROTO(unsigned int function, unsigned int index, unsigned long rax, + unsigned long rbx, unsigned long rcx, unsigned long rdx, + bool found, bool used_max_basic), + TP_ARGS(function, index, rax, rbx, rcx, rdx, found, used_max_basic), TP_STRUCT__entry( __field( unsigned int, function ) + __field( unsigned int, index ) __field( unsigned long, rax ) __field( unsigned long, rbx ) __field( unsigned long, rcx ) __field( unsigned long, rdx ) __field( bool, found ) + __field( bool, used_max_basic ) ), TP_fast_assign( __entry->function = function; + __entry->index = index; __entry->rax = rax; __entry->rbx = rbx; __entry->rcx = rcx; __entry->rdx = rdx; __entry->found = found; + __entry->used_max_basic = used_max_basic; ), - TP_printk("func %x rax %lx rbx %lx rcx %lx rdx %lx, cpuid entry %s", - __entry->function, __entry->rax, + TP_printk("func %x idx %x rax %lx rbx %lx rcx %lx rdx %lx, cpuid entry %s%s", + __entry->function, __entry->index, __entry->rax, __entry->rbx, __entry->rcx, __entry->rdx, - __entry->found ? "found" : "not found") + __entry->found ? "found" : "not found", + __entry->used_max_basic ? ", used max basic" : "") ); #define AREG(x) { APIC_##x, "APIC_" #x } @@ -240,7 +246,7 @@ TRACE_EVENT(kvm_exit, __entry->guest_rip = kvm_rip_read(vcpu); __entry->isa = isa; __entry->vcpu_id = vcpu->vcpu_id; - kvm_x86_ops->get_exit_info(vcpu, &__entry->info1, + kvm_x86_ops.get_exit_info(vcpu, &__entry->info1, &__entry->info2); ), @@ -744,14 +750,14 @@ TRACE_EVENT(kvm_emulate_insn, ), TP_fast_assign( - __entry->csbase = kvm_x86_ops->get_segment_base(vcpu, VCPU_SREG_CS); - __entry->len = vcpu->arch.emulate_ctxt.fetch.ptr - - vcpu->arch.emulate_ctxt.fetch.data; - __entry->rip = vcpu->arch.emulate_ctxt._eip - __entry->len; + __entry->csbase = kvm_x86_ops.get_segment_base(vcpu, VCPU_SREG_CS); + __entry->len = vcpu->arch.emulate_ctxt->fetch.ptr + - vcpu->arch.emulate_ctxt->fetch.data; + __entry->rip = vcpu->arch.emulate_ctxt->_eip - __entry->len; memcpy(__entry->insn, - vcpu->arch.emulate_ctxt.fetch.data, + vcpu->arch.emulate_ctxt->fetch.data, 15); - __entry->flags = kei_decode_mode(vcpu->arch.emulate_ctxt.mode); + __entry->flags = kei_decode_mode(vcpu->arch.emulate_ctxt->mode); __entry->failed = failed; ), @@ -1367,6 +1373,24 @@ TRACE_EVENT(kvm_avic_unaccelerated_access, __entry->vec) ); +TRACE_EVENT(kvm_avic_ga_log, + TP_PROTO(u32 vmid, u32 vcpuid), + TP_ARGS(vmid, vcpuid), + + TP_STRUCT__entry( + __field(u32, vmid) + __field(u32, vcpuid) + ), + + TP_fast_assign( + __entry->vmid = vmid; + __entry->vcpuid = vcpuid; + ), + + TP_printk("vmid=%u, vcpuid=%u", + __entry->vmid, __entry->vcpuid) +); + TRACE_EVENT(kvm_hv_timer_state, TP_PROTO(unsigned int vcpu_id, unsigned int hv_timer_in_use), TP_ARGS(vcpu_id, hv_timer_in_use), diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index f486e2606247..8903475f751e 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -101,7 +101,7 @@ static inline bool cpu_has_load_perf_global_ctrl(void) (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL); } -static inline bool vmx_mpx_supported(void) +static inline bool cpu_has_vmx_mpx(void) { return (vmcs_config.vmexit_ctrl & VM_EXIT_CLEAR_BNDCFGS) && (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_BNDCFGS); @@ -146,11 +146,6 @@ static inline bool vmx_umip_emulated(void) SECONDARY_EXEC_DESC; } -static inline bool vmx_pku_supported(void) -{ - return boot_cpu_has(X86_FEATURE_PKU); -} - static inline bool cpu_has_vmx_rdtscp(void) { return vmcs_config.cpu_based_2nd_exec_ctrl & @@ -354,4 +349,22 @@ static inline bool cpu_has_vmx_intel_pt(void) (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_RTIT_CTL); } +/* + * Processor Trace can operate in one of three modes: + * a. system-wide: trace both host/guest and output to host buffer + * b. host-only: only trace host and output to host buffer + * c. host-guest: trace host and guest simultaneously and output to their + * respective buffer + * + * KVM currently only supports (a) and (c). + */ +static inline bool vmx_pt_mode_is_system(void) +{ + return pt_mode == PT_MODE_SYSTEM; +} +static inline bool vmx_pt_mode_is_host_guest(void) +{ + return pt_mode == PT_MODE_HOST_GUEST; +} + #endif /* __KVM_X86_VMX_CAPS_H */ diff --git a/arch/x86/kvm/vmx/evmcs.h b/arch/x86/kvm/vmx/evmcs.h index 6de47f2569c9..e5f7a7ebf27d 100644 --- a/arch/x86/kvm/vmx/evmcs.h +++ b/arch/x86/kvm/vmx/evmcs.h @@ -198,6 +198,13 @@ static inline void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) {} static inline void evmcs_touch_msr_bitmap(void) {} #endif /* IS_ENABLED(CONFIG_HYPERV) */ +enum nested_evmptrld_status { + EVMPTRLD_DISABLED, + EVMPTRLD_SUCCEEDED, + EVMPTRLD_VMFAIL, + EVMPTRLD_ERROR, +}; + bool nested_enlightened_vmentry(struct kvm_vcpu *vcpu, u64 *evmcs_gpa); uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu); int nested_enable_evmcs(struct kvm_vcpu *vcpu, diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 9750e590c89d..de232306561a 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -353,9 +353,8 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) to_vmx(vcpu)->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT, nested_ept_ad_enabled(vcpu), - nested_ept_get_cr3(vcpu)); - vcpu->arch.mmu->set_cr3 = vmx_set_cr3; - vcpu->arch.mmu->get_cr3 = nested_ept_get_cr3; + nested_ept_get_eptp(vcpu)); + vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp; vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; @@ -1910,18 +1909,18 @@ static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) * This is an equivalent of the nested hypervisor executing the vmptrld * instruction. */ -static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu, - bool from_launch) +static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( + struct kvm_vcpu *vcpu, bool from_launch) { struct vcpu_vmx *vmx = to_vmx(vcpu); bool evmcs_gpa_changed = false; u64 evmcs_gpa; if (likely(!vmx->nested.enlightened_vmcs_enabled)) - return 1; + return EVMPTRLD_DISABLED; if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa)) - return 1; + return EVMPTRLD_DISABLED; if (unlikely(!vmx->nested.hv_evmcs || evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { @@ -1932,7 +1931,7 @@ static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu, if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa), &vmx->nested.hv_evmcs_map)) - return 0; + return EVMPTRLD_ERROR; vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; @@ -1961,7 +1960,7 @@ static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu, if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { nested_release_evmcs(vcpu); - return 0; + return EVMPTRLD_VMFAIL; } vmx->nested.dirty_vmcs12 = true; @@ -1990,21 +1989,13 @@ static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu, vmx->nested.hv_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; - return 1; + return EVMPTRLD_SUCCEEDED; } void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - /* - * hv_evmcs may end up being not mapped after migration (when - * L2 was running), map it here to make sure vmcs12 changes are - * properly reflected. - */ - if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) - nested_vmx_handle_enlightened_vmptrld(vcpu, false); - if (vmx->nested.hv_evmcs) { copy_vmcs12_to_enlightened(vmx); /* All fields are clean */ @@ -2475,9 +2466,9 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, * If L1 use EPT, then L0 needs to execute INVEPT on * EPTP02 instead of EPTP01. Therefore, delay TLB * flush until vmcs02->eptp is fully updated by - * KVM_REQ_LOAD_CR3. Note that this assumes + * KVM_REQ_LOAD_MMU_PGD. Note that this assumes * KVM_REQ_TLB_FLUSH is evaluated after - * KVM_REQ_LOAD_CR3 in vcpu_enter_guest(). + * KVM_REQ_LOAD_MMU_PGD in vcpu_enter_guest(). */ kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); } @@ -2522,7 +2513,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, /* * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12 * on nested VM-Exit, which can occur without actually running L2 and - * thus without hitting vmx_set_cr3(), e.g. if L1 is entering L2 with + * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the * transition to HLT instead of running L2. */ @@ -2564,13 +2555,13 @@ static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) return 0; } -static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) +static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) { struct vcpu_vmx *vmx = to_vmx(vcpu); int maxphyaddr = cpuid_maxphyaddr(vcpu); /* Check for memory type validity */ - switch (address & VMX_EPTP_MT_MASK) { + switch (new_eptp & VMX_EPTP_MT_MASK) { case VMX_EPTP_MT_UC: if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) return false; @@ -2583,16 +2574,26 @@ static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) return false; } - /* only 4 levels page-walk length are valid */ - if (CC((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4)) + /* Page-walk levels validity. */ + switch (new_eptp & VMX_EPTP_PWL_MASK) { + case VMX_EPTP_PWL_5: + if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT))) + return false; + break; + case VMX_EPTP_PWL_4: + if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT))) + return false; + break; + default: return false; + } /* Reserved bits should not be set */ - if (CC(address >> maxphyaddr || ((address >> 7) & 0x1f))) + if (CC(new_eptp >> maxphyaddr || ((new_eptp >> 7) & 0x1f))) return false; /* AD, if set, should be supported */ - if (address & VMX_EPTP_AD_ENABLE_BIT) { + if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) { if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) return false; } @@ -2641,7 +2642,7 @@ static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, return -EINVAL; if (nested_cpu_has_ept(vmcs12) && - CC(!valid_ept_address(vcpu, vmcs12->ept_pointer))) + CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer))) return -EINVAL; if (nested_cpu_has_vmfunc(vmcs12)) { @@ -2961,7 +2962,7 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) /* * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to - * be written (by preparve_vmcs02()) before the "real" VMEnter, i.e. + * be written (by prepare_vmcs02()) before the "real" VMEnter, i.e. * there is no need to preserve other bits or save/restore the field. */ vmcs_writel(GUEST_RFLAGS, 0); @@ -3053,6 +3054,27 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) struct page *page; u64 hpa; + /* + * hv_evmcs may end up being not mapped after migration (when + * L2 was running), map it here to make sure vmcs12 changes are + * properly reflected. + */ + if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) { + enum nested_evmptrld_status evmptrld_status = + nested_vmx_handle_enlightened_vmptrld(vcpu, false); + + if (evmptrld_status == EVMPTRLD_VMFAIL || + evmptrld_status == EVMPTRLD_ERROR) { + pr_debug_ratelimited("%s: enlightened vmptrld failed\n", + __func__); + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->internal.suberror = + KVM_INTERNAL_ERROR_EMULATION; + vcpu->run->internal.ndata = 0; + return false; + } + } + if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { /* * Translate L1 physical address to host physical @@ -3316,12 +3338,18 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) enum nvmx_vmentry_status status; struct vcpu_vmx *vmx = to_vmx(vcpu); u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); + enum nested_evmptrld_status evmptrld_status; if (!nested_vmx_check_permission(vcpu)) return 1; - if (!nested_vmx_handle_enlightened_vmptrld(vcpu, launch)) + evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, launch); + if (evmptrld_status == EVMPTRLD_ERROR) { + kvm_queue_exception(vcpu, UD_VECTOR); return 1; + } else if (evmptrld_status == EVMPTRLD_VMFAIL) { + return nested_vmx_failInvalid(vcpu); + } if (!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull) return nested_vmx_failInvalid(vcpu); @@ -3499,7 +3527,7 @@ static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, } -static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) +void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); gfn_t gfn; @@ -3604,7 +3632,7 @@ static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) vcpu->arch.exception.payload); } -static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr) +static int vmx_check_nested_events(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); unsigned long exit_qual; @@ -3680,8 +3708,7 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr) return 0; } - if ((kvm_cpu_has_interrupt(vcpu) || external_intr) && - nested_exit_on_intr(vcpu)) { + if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(vcpu)) { if (block_nested_events) return -EBUSY; nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); @@ -4024,7 +4051,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, * * If vmcs12 uses EPT, we need to execute this flush on EPTP01 * and therefore we request the TLB flush to happen only after VMCS EPTP - * has been set by KVM_REQ_LOAD_CR3. + * has been set by KVM_REQ_LOAD_MMU_PGD. */ if (enable_vpid && (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) { @@ -4329,17 +4356,8 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; if (likely(!vmx->fail)) { - /* - * TODO: SDM says that with acknowledge interrupt on - * exit, bit 31 of the VM-exit interrupt information - * (valid interrupt) is always set to 1 on - * EXIT_REASON_EXTERNAL_INTERRUPT, so we shouldn't - * need kvm_cpu_has_interrupt(). See the commit - * message for details. - */ - if (nested_exit_intr_ack_set(vcpu) && - exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && - kvm_cpu_has_interrupt(vcpu)) { + if (exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && + nested_exit_intr_ack_set(vcpu)) { int irq = kvm_cpu_get_interrupt(vcpu); WARN_ON(irq < 0); vmcs12->vm_exit_intr_info = irq | @@ -4383,7 +4401,7 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, * Decode the memory-address operand of a vmx instruction, as recorded on an * exit caused by such an instruction (run by a guest hypervisor). * On success, returns 0. When the operand is invalid, returns 1 and throws - * #UD or #GP. + * #UD, #GP, or #SS. */ int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, u32 vmx_instruction_info, bool wr, int len, gva_t *ret) @@ -4424,7 +4442,7 @@ int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, if (base_is_valid) off += kvm_register_read(vcpu, base_reg); if (index_is_valid) - off += kvm_register_read(vcpu, index_reg)<pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) { + if (kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) { vmx->nested.msrs.entry_ctls_high |= VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; vmx->nested.msrs.exit_ctls_high |= @@ -4603,7 +4621,7 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu) vmx->nested.vmcs02_initialized = false; vmx->nested.vmxon = true; - if (pt_mode == PT_MODE_HOST_GUEST) { + if (vmx_pt_mode_is_host_guest()) { vmx->pt_desc.guest.ctl = 0; pt_update_intercept_for_msr(vmx); } @@ -5235,7 +5253,7 @@ static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { u32 index = kvm_rcx_read(vcpu); - u64 address; + u64 new_eptp; bool accessed_dirty; struct kvm_mmu *mmu = vcpu->arch.walk_mmu; @@ -5248,23 +5266,23 @@ static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, - &address, index * 8, 8)) + &new_eptp, index * 8, 8)) return 1; - accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT); + accessed_dirty = !!(new_eptp & VMX_EPTP_AD_ENABLE_BIT); /* * If the (L2) guest does a vmfunc to the currently * active ept pointer, we don't have to do anything else */ - if (vmcs12->ept_pointer != address) { - if (!valid_ept_address(vcpu, address)) + if (vmcs12->ept_pointer != new_eptp) { + if (!nested_vmx_check_eptp(vcpu, new_eptp)) return 1; kvm_mmu_unload(vcpu); mmu->ept_ad = accessed_dirty; mmu->mmu_role.base.ad_disabled = !accessed_dirty; - vmcs12->ept_pointer = address; + vmcs12->ept_pointer = new_eptp; /* * TODO: Check what's the correct approach in case * mmu reload fails. Currently, we just let the next @@ -5525,8 +5543,7 @@ bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - if (vmx->nested.nested_run_pending) - return false; + WARN_ON_ONCE(vmx->nested.nested_run_pending); if (unlikely(vmx->fail)) { trace_kvm_nested_vmenter_failed( @@ -5535,19 +5552,6 @@ bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) return true; } - /* - * The host physical addresses of some pages of guest memory - * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC - * Page). The CPU may write to these pages via their host - * physical address while L2 is running, bypassing any - * address-translation-based dirty tracking (e.g. EPT write - * protection). - * - * Mark them dirty on every exit from L2 to prevent them from - * getting out of sync with dirty tracking. - */ - nested_mark_vmcs12_pages_dirty(vcpu); - trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, vmcs_readl(EXIT_QUALIFICATION), vmx->idt_vectoring_info, @@ -5628,7 +5632,7 @@ bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) case EXIT_REASON_MWAIT_INSTRUCTION: return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); case EXIT_REASON_MONITOR_TRAP_FLAG: - return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG); + return nested_cpu_has_mtf(vmcs12); case EXIT_REASON_MONITOR_INSTRUCTION: return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); case EXIT_REASON_PAUSE_INSTRUCTION: @@ -5905,10 +5909,12 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa); } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { /* - * Sync eVMCS upon entry as we may not have - * HV_X64_MSR_VP_ASSIST_PAGE set up yet. + * nested_vmx_handle_enlightened_vmptrld() cannot be called + * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be + * restored yet. EVMCS will be mapped from + * nested_get_vmcs12_pages(). */ - vmx->nested.need_vmcs12_to_shadow_sync = true; + kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu); } else { return -EINVAL; } @@ -6130,11 +6136,13 @@ void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) /* nested EPT: emulate EPT also to L1 */ msrs->secondary_ctls_high |= SECONDARY_EXEC_ENABLE_EPT; - msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT | - VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT; - if (cpu_has_vmx_ept_execute_only()) - msrs->ept_caps |= - VMX_EPT_EXECUTE_ONLY_BIT; + msrs->ept_caps = + VMX_EPT_PAGE_WALK_4_BIT | + VMX_EPT_PAGE_WALK_5_BIT | + VMX_EPTP_WB_BIT | + VMX_EPT_INVEPT_BIT | + VMX_EPT_EXECUTE_ONLY_BIT; + msrs->ept_caps &= ept_caps; msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | @@ -6233,7 +6241,8 @@ void nested_vmx_hardware_unsetup(void) } } -__init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) +__init int nested_vmx_hardware_setup(struct kvm_x86_ops *ops, + int (*exit_handlers[])(struct kvm_vcpu *)) { int i; @@ -6269,12 +6278,12 @@ __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid; exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc; - kvm_x86_ops->check_nested_events = vmx_check_nested_events; - kvm_x86_ops->get_nested_state = vmx_get_nested_state; - kvm_x86_ops->set_nested_state = vmx_set_nested_state; - kvm_x86_ops->get_vmcs12_pages = nested_get_vmcs12_pages; - kvm_x86_ops->nested_enable_evmcs = nested_enable_evmcs; - kvm_x86_ops->nested_get_evmcs_version = nested_get_evmcs_version; + ops->check_nested_events = vmx_check_nested_events; + ops->get_nested_state = vmx_get_nested_state; + ops->set_nested_state = vmx_set_nested_state; + ops->get_vmcs12_pages = nested_get_vmcs12_pages; + ops->nested_enable_evmcs = nested_enable_evmcs; + ops->nested_get_evmcs_version = nested_get_evmcs_version; return 0; } diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index 9aeda46f473e..ac56aefa49e3 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -19,7 +19,8 @@ enum nvmx_vmentry_status { void vmx_leave_nested(struct kvm_vcpu *vcpu); void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps); void nested_vmx_hardware_unsetup(void); -__init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)); +__init int nested_vmx_hardware_setup(struct kvm_x86_ops *ops, + int (*exit_handlers[])(struct kvm_vcpu *)); void nested_vmx_set_vmcs_shadowing_bitmap(void); void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu); enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, @@ -33,6 +34,7 @@ int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata); int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, u32 vmx_instruction_info, bool wr, int len, gva_t *ret); void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu); +void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu); bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port, int size); @@ -60,7 +62,7 @@ static inline int vmx_has_valid_vmcs12(struct kvm_vcpu *vcpu) vmx->nested.hv_evmcs; } -static inline unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu) +static inline unsigned long nested_ept_get_eptp(struct kvm_vcpu *vcpu) { /* return the page table to be shadowed - in our case, EPT12 */ return get_vmcs12(vcpu)->ept_pointer; @@ -68,7 +70,7 @@ static inline unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu) static inline bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu) { - return nested_ept_get_cr3(vcpu) & VMX_EPTP_AD_ENABLE_BIT; + return nested_ept_get_eptp(vcpu) & VMX_EPTP_AD_ENABLE_BIT; } /* diff --git a/arch/x86/kvm/vmx/ops.h b/arch/x86/kvm/vmx/ops.h index 45eaedee2ac0..19717d0a1100 100644 --- a/arch/x86/kvm/vmx/ops.h +++ b/arch/x86/kvm/vmx/ops.h @@ -13,6 +13,8 @@ #define __ex(x) __kvm_handle_fault_on_reboot(x) asmlinkage void vmread_error(unsigned long field, bool fault); +__attribute__((regparm(0))) void vmread_error_trampoline(unsigned long field, + bool fault); void vmwrite_error(unsigned long field, unsigned long value); void vmclear_error(struct vmcs *vmcs, u64 phys_addr); void vmptrld_error(struct vmcs *vmcs, u64 phys_addr); @@ -70,15 +72,28 @@ static __always_inline unsigned long __vmcs_readl(unsigned long field) asm volatile("1: vmread %2, %1\n\t" ".byte 0x3e\n\t" /* branch taken hint */ "ja 3f\n\t" - "mov %2, %%" _ASM_ARG1 "\n\t" - "xor %%" _ASM_ARG2 ", %%" _ASM_ARG2 "\n\t" - "2: call vmread_error\n\t" - "xor %k1, %k1\n\t" + + /* + * VMREAD failed. Push '0' for @fault, push the failing + * @field, and bounce through the trampoline to preserve + * volatile registers. + */ + "push $0\n\t" + "push %2\n\t" + "2:call vmread_error_trampoline\n\t" + + /* + * Unwind the stack. Note, the trampoline zeros out the + * memory for @fault so that the result is '0' on error. + */ + "pop %2\n\t" + "pop %1\n\t" "3:\n\t" + /* VMREAD faulted. As above, except push '1' for @fault. */ ".pushsection .fixup, \"ax\"\n\t" - "4: mov %2, %%" _ASM_ARG1 "\n\t" - "mov $1, %%" _ASM_ARG2 "\n\t" + "4: push $1\n\t" + "push %2\n\t" "jmp 2b\n\t" ".popsection\n\t" _ASM_EXTABLE(1b, 4b) diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index fd21cdb10b79..7c857737b438 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -263,9 +263,15 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!msr_info->host_initiated) data = (s64)(s32)data; pmc->counter += data - pmc_read_counter(pmc); + if (pmc->perf_event) + perf_event_period(pmc->perf_event, + get_sample_period(pmc, data)); return 0; } else if ((pmc = get_fixed_pmc(pmu, msr))) { pmc->counter += data - pmc_read_counter(pmc); + if (pmc->perf_event) + perf_event_period(pmc->perf_event, + get_sample_period(pmc, data)); return 0; } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { if (data == pmc->eventsel) @@ -329,7 +335,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->global_ovf_ctrl_mask = pmu->global_ctrl_mask & ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF | MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD); - if (kvm_x86_ops->pt_supported()) + if (vmx_pt_mode_is_host_guest()) pmu->global_ovf_ctrl_mask &= ~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI; diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index 81ada2ce99e7..9651ba388ba9 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -135,12 +135,12 @@ SYM_FUNC_START(__vmx_vcpu_run) cmpb $0, %bl /* Load guest registers. Don't clobber flags. */ - mov VCPU_RBX(%_ASM_AX), %_ASM_BX mov VCPU_RCX(%_ASM_AX), %_ASM_CX mov VCPU_RDX(%_ASM_AX), %_ASM_DX + mov VCPU_RBX(%_ASM_AX), %_ASM_BX + mov VCPU_RBP(%_ASM_AX), %_ASM_BP mov VCPU_RSI(%_ASM_AX), %_ASM_SI mov VCPU_RDI(%_ASM_AX), %_ASM_DI - mov VCPU_RBP(%_ASM_AX), %_ASM_BP #ifdef CONFIG_X86_64 mov VCPU_R8 (%_ASM_AX), %r8 mov VCPU_R9 (%_ASM_AX), %r9 @@ -168,12 +168,12 @@ SYM_FUNC_START(__vmx_vcpu_run) /* Save all guest registers, including RAX from the stack */ __ASM_SIZE(pop) VCPU_RAX(%_ASM_AX) - mov %_ASM_BX, VCPU_RBX(%_ASM_AX) mov %_ASM_CX, VCPU_RCX(%_ASM_AX) mov %_ASM_DX, VCPU_RDX(%_ASM_AX) + mov %_ASM_BX, VCPU_RBX(%_ASM_AX) + mov %_ASM_BP, VCPU_RBP(%_ASM_AX) mov %_ASM_SI, VCPU_RSI(%_ASM_AX) mov %_ASM_DI, VCPU_RDI(%_ASM_AX) - mov %_ASM_BP, VCPU_RBP(%_ASM_AX) #ifdef CONFIG_X86_64 mov %r8, VCPU_R8 (%_ASM_AX) mov %r9, VCPU_R9 (%_ASM_AX) @@ -197,12 +197,12 @@ SYM_FUNC_START(__vmx_vcpu_run) * free. RSP and RAX are exempt as RSP is restored by hardware during * VM-Exit and RAX is explicitly loaded with 0 or 1 to return VM-Fail. */ -1: xor %ebx, %ebx - xor %ecx, %ecx +1: xor %ecx, %ecx xor %edx, %edx + xor %ebx, %ebx + xor %ebp, %ebp xor %esi, %esi xor %edi, %edi - xor %ebp, %ebp #ifdef CONFIG_X86_64 xor %r8d, %r8d xor %r9d, %r9d @@ -234,3 +234,61 @@ SYM_FUNC_START(__vmx_vcpu_run) 2: mov $1, %eax jmp 1b SYM_FUNC_END(__vmx_vcpu_run) + +/** + * vmread_error_trampoline - Trampoline from inline asm to vmread_error() + * @field: VMCS field encoding that failed + * @fault: %true if the VMREAD faulted, %false if it failed + + * Save and restore volatile registers across a call to vmread_error(). Note, + * all parameters are passed on the stack. + */ +SYM_FUNC_START(vmread_error_trampoline) + push %_ASM_BP + mov %_ASM_SP, %_ASM_BP + + push %_ASM_AX + push %_ASM_CX + push %_ASM_DX +#ifdef CONFIG_X86_64 + push %rdi + push %rsi + push %r8 + push %r9 + push %r10 + push %r11 +#endif +#ifdef CONFIG_X86_64 + /* Load @field and @fault to arg1 and arg2 respectively. */ + mov 3*WORD_SIZE(%rbp), %_ASM_ARG2 + mov 2*WORD_SIZE(%rbp), %_ASM_ARG1 +#else + /* Parameters are passed on the stack for 32-bit (see asmlinkage). */ + push 3*WORD_SIZE(%ebp) + push 2*WORD_SIZE(%ebp) +#endif + + call vmread_error + +#ifndef CONFIG_X86_64 + add $8, %esp +#endif + + /* Zero out @fault, which will be popped into the result register. */ + _ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP) + +#ifdef CONFIG_X86_64 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rsi + pop %rdi +#endif + pop %_ASM_DX + pop %_ASM_CX + pop %_ASM_AX + pop %_ASM_BP + + ret +SYM_FUNC_END(vmread_error_trampoline) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 458e684dfbdc..91749f1254e8 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -437,7 +437,6 @@ static const struct kvm_vmx_segment_field { VMX_SEGMENT_FIELD(LDTR), }; -u64 host_efer; static unsigned long host_idt_base; /* @@ -658,53 +657,16 @@ static int vmx_set_guest_msr(struct vcpu_vmx *vmx, struct shared_msr_entry *msr, return ret; } -void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs) -{ - vmcs_clear(loaded_vmcs->vmcs); - if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) - vmcs_clear(loaded_vmcs->shadow_vmcs); - loaded_vmcs->cpu = -1; - loaded_vmcs->launched = 0; -} - #ifdef CONFIG_KEXEC_CORE -/* - * This bitmap is used to indicate whether the vmclear - * operation is enabled on all cpus. All disabled by - * default. - */ -static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE; - -static inline void crash_enable_local_vmclear(int cpu) -{ - cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap); -} - -static inline void crash_disable_local_vmclear(int cpu) -{ - cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap); -} - -static inline int crash_local_vmclear_enabled(int cpu) -{ - return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap); -} - static void crash_vmclear_local_loaded_vmcss(void) { int cpu = raw_smp_processor_id(); struct loaded_vmcs *v; - if (!crash_local_vmclear_enabled(cpu)) - return; - list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), loaded_vmcss_on_cpu_link) vmcs_clear(v->vmcs); } -#else -static inline void crash_enable_local_vmclear(int cpu) { } -static inline void crash_disable_local_vmclear(int cpu) { } #endif /* CONFIG_KEXEC_CORE */ static void __loaded_vmcs_clear(void *arg) @@ -716,19 +678,24 @@ static void __loaded_vmcs_clear(void *arg) return; /* vcpu migration can race with cpu offline */ if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs) per_cpu(current_vmcs, cpu) = NULL; - crash_disable_local_vmclear(cpu); + + vmcs_clear(loaded_vmcs->vmcs); + if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) + vmcs_clear(loaded_vmcs->shadow_vmcs); + list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link); /* - * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link - * is before setting loaded_vmcs->vcpu to -1 which is done in - * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist - * then adds the vmcs into percpu list before it is deleted. + * Ensure all writes to loaded_vmcs, including deleting it from its + * current percpu list, complete before setting loaded_vmcs->vcpu to + * -1, otherwise a different cpu can see vcpu == -1 first and add + * loaded_vmcs to its percpu list before it's deleted from this cpu's + * list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs(). */ smp_wmb(); - loaded_vmcs_init(loaded_vmcs); - crash_enable_local_vmclear(cpu); + loaded_vmcs->cpu = -1; + loaded_vmcs->launched = 0; } void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) @@ -812,7 +779,7 @@ void update_exception_bitmap(struct kvm_vcpu *vcpu) if (to_vmx(vcpu)->rmode.vm86_active) eb = ~0; if (enable_ept) - eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */ + eb &= ~(1u << PF_VECTOR); /* When we are running a nested L2 guest and L1 specified for it a * certain exception bitmap, we must trap the same exceptions and pass @@ -1063,7 +1030,7 @@ static unsigned long segment_base(u16 selector) static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) { - return (pt_mode == PT_MODE_HOST_GUEST) && + return vmx_pt_mode_is_host_guest() && !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); } @@ -1097,7 +1064,7 @@ static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range) static void pt_guest_enter(struct vcpu_vmx *vmx) { - if (pt_mode == PT_MODE_SYSTEM) + if (vmx_pt_mode_is_system()) return; /* @@ -1114,7 +1081,7 @@ static void pt_guest_enter(struct vcpu_vmx *vmx) static void pt_guest_exit(struct vcpu_vmx *vmx) { - if (pt_mode == PT_MODE_SYSTEM) + if (vmx_pt_mode_is_system()) return; if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { @@ -1347,18 +1314,17 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu) if (!already_loaded) { loaded_vmcs_clear(vmx->loaded_vmcs); local_irq_disable(); - crash_disable_local_vmclear(cpu); /* - * Read loaded_vmcs->cpu should be before fetching - * loaded_vmcs->loaded_vmcss_on_cpu_link. - * See the comments in __loaded_vmcs_clear(). + * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to + * this cpu's percpu list, otherwise it may not yet be deleted + * from its previous cpu's percpu list. Pairs with the + * smb_wmb() in __loaded_vmcs_clear(). */ smp_rmb(); list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link, &per_cpu(loaded_vmcss_on_cpu, cpu)); - crash_enable_local_vmclear(cpu); local_irq_enable(); } @@ -1691,16 +1657,6 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu) vmx_clear_hlt(vcpu); } -static bool vmx_rdtscp_supported(void) -{ - return cpu_has_vmx_rdtscp(); -} - -static bool vmx_invpcid_supported(void) -{ - return cpu_has_vmx_invpcid(); -} - /* * Swap MSR entry in host/guest MSR entry array. */ @@ -1908,24 +1864,24 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) &msr_info->data); break; case MSR_IA32_RTIT_CTL: - if (pt_mode != PT_MODE_HOST_GUEST) + if (!vmx_pt_mode_is_host_guest()) return 1; msr_info->data = vmx->pt_desc.guest.ctl; break; case MSR_IA32_RTIT_STATUS: - if (pt_mode != PT_MODE_HOST_GUEST) + if (!vmx_pt_mode_is_host_guest()) return 1; msr_info->data = vmx->pt_desc.guest.status; break; case MSR_IA32_RTIT_CR3_MATCH: - if ((pt_mode != PT_MODE_HOST_GUEST) || + if (!vmx_pt_mode_is_host_guest() || !intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering)) return 1; msr_info->data = vmx->pt_desc.guest.cr3_match; break; case MSR_IA32_RTIT_OUTPUT_BASE: - if ((pt_mode != PT_MODE_HOST_GUEST) || + if (!vmx_pt_mode_is_host_guest() || (!intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output) && !intel_pt_validate_cap(vmx->pt_desc.caps, @@ -1934,7 +1890,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) msr_info->data = vmx->pt_desc.guest.output_base; break; case MSR_IA32_RTIT_OUTPUT_MASK: - if ((pt_mode != PT_MODE_HOST_GUEST) || + if (!vmx_pt_mode_is_host_guest() || (!intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output) && !intel_pt_validate_cap(vmx->pt_desc.caps, @@ -1944,7 +1900,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; - if ((pt_mode != PT_MODE_HOST_GUEST) || + if (!vmx_pt_mode_is_host_guest() || (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_num_address_ranges))) return 1; @@ -2150,7 +2106,7 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 1; return vmx_set_vmx_msr(vcpu, msr_index, data); case MSR_IA32_RTIT_CTL: - if ((pt_mode != PT_MODE_HOST_GUEST) || + if (!vmx_pt_mode_is_host_guest() || vmx_rtit_ctl_check(vcpu, data) || vmx->nested.vmxon) return 1; @@ -2266,18 +2222,33 @@ static __init int vmx_disabled_by_bios(void) !boot_cpu_has(X86_FEATURE_VMX); } -static void kvm_cpu_vmxon(u64 addr) +static int kvm_cpu_vmxon(u64 vmxon_pointer) { + u64 msr; + cr4_set_bits(X86_CR4_VMXE); intel_pt_handle_vmx(1); - asm volatile ("vmxon %0" : : "m"(addr)); + asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t" + _ASM_EXTABLE(1b, %l[fault]) + : : [vmxon_pointer] "m"(vmxon_pointer) + : : fault); + return 0; + +fault: + WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", + rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); + intel_pt_handle_vmx(0); + cr4_clear_bits(X86_CR4_VMXE); + + return -EFAULT; } static int hardware_enable(void) { int cpu = raw_smp_processor_id(); u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); + int r; if (cr4_read_shadow() & X86_CR4_VMXE) return -EBUSY; @@ -2294,18 +2265,10 @@ static int hardware_enable(void) INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu)); spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); - /* - * Now we can enable the vmclear operation in kdump - * since the loaded_vmcss_on_cpu list on this cpu - * has been initialized. - * - * Though the cpu is not in VMX operation now, there - * is no problem to enable the vmclear operation - * for the loaded_vmcss_on_cpu list is empty! - */ - crash_enable_local_vmclear(cpu); + r = kvm_cpu_vmxon(phys_addr); + if (r) + return r; - kvm_cpu_vmxon(phys_addr); if (enable_ept) ept_sync_global(); @@ -2617,9 +2580,12 @@ int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) if (!loaded_vmcs->vmcs) return -ENOMEM; + vmcs_clear(loaded_vmcs->vmcs); + loaded_vmcs->shadow_vmcs = NULL; loaded_vmcs->hv_timer_soft_disabled = false; - loaded_vmcs_init(loaded_vmcs); + loaded_vmcs->cpu = -1; + loaded_vmcs->launched = 0; if (cpu_has_vmx_msr_bitmap()) { loaded_vmcs->msr_bitmap = (unsigned long *) @@ -3001,9 +2967,8 @@ void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) static int get_ept_level(struct kvm_vcpu *vcpu) { - /* Nested EPT currently only supports 4-level walks. */ if (is_guest_mode(vcpu) && nested_cpu_has_ept(get_vmcs12(vcpu))) - return 4; + return vmx_eptp_page_walk_level(nested_ept_get_eptp(vcpu)); if (cpu_has_vmx_ept_5levels() && (cpuid_maxphyaddr(vcpu) > 48)) return 5; return 4; @@ -3023,7 +2988,7 @@ u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa) return eptp; } -void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long cr3) { struct kvm *kvm = vcpu->kvm; bool update_guest_cr3 = true; @@ -3035,7 +3000,7 @@ void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) eptp = construct_eptp(vcpu, cr3); vmcs_write64(EPT_POINTER, eptp); - if (kvm_x86_ops->tlb_remote_flush) { + if (kvm_x86_ops.tlb_remote_flush) { spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock); to_vmx(vcpu)->ept_pointer = eptp; to_kvm_vmx(kvm)->ept_pointers_match @@ -4040,7 +4005,7 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; - if (pt_mode == PT_MODE_SYSTEM) + if (vmx_pt_mode_is_system()) exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX); if (!cpu_need_virtualize_apic_accesses(vcpu)) exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; @@ -4095,7 +4060,7 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) } } - if (vmx_rdtscp_supported()) { + if (cpu_has_vmx_rdtscp()) { bool rdtscp_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP); if (!rdtscp_enabled) exec_control &= ~SECONDARY_EXEC_RDTSCP; @@ -4110,7 +4075,7 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) } } - if (vmx_invpcid_supported()) { + if (cpu_has_vmx_invpcid()) { /* Exposing INVPCID only when PCID is exposed */ bool invpcid_enabled = guest_cpuid_has(vcpu, X86_FEATURE_INVPCID) && @@ -4281,7 +4246,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) if (cpu_has_vmx_encls_vmexit()) vmcs_write64(ENCLS_EXITING_BITMAP, -1ull); - if (pt_mode == PT_MODE_HOST_GUEST) { + if (vmx_pt_mode_is_host_guest()) { memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); /* Bit[6~0] are forced to 1, writes are ignored. */ vmx->pt_desc.guest.output_mask = 0x7F; @@ -4509,8 +4474,13 @@ static int vmx_nmi_allowed(struct kvm_vcpu *vcpu) static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu) { - return (!to_vmx(vcpu)->nested.nested_run_pending && - vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && + if (to_vmx(vcpu)->nested.nested_run_pending) + return false; + + if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) + return true; + + return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); } @@ -4566,7 +4536,6 @@ static bool rmode_exception(struct kvm_vcpu *vcpu, int vec) case GP_VECTOR: case MF_VECTOR: return true; - break; } return false; } @@ -5343,7 +5312,6 @@ static void vmx_enable_tdp(void) VMX_EPT_RWX_MASK, 0ull); ept_set_mmio_spte_mask(); - kvm_enable_tdp(); } /* @@ -5876,8 +5844,23 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, if (vmx->emulation_required) return handle_invalid_guest_state(vcpu); - if (is_guest_mode(vcpu) && nested_vmx_exit_reflected(vcpu, exit_reason)) - return nested_vmx_reflect_vmexit(vcpu, exit_reason); + if (is_guest_mode(vcpu)) { + /* + * The host physical addresses of some pages of guest memory + * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC + * Page). The CPU may write to these pages via their host + * physical address while L2 is running, bypassing any + * address-translation-based dirty tracking (e.g. EPT write + * protection). + * + * Mark them dirty on every exit from L2 to prevent them from + * getting out of sync with dirty tracking. + */ + nested_mark_vmcs12_pages_dirty(vcpu); + + if (nested_vmx_exit_reflected(vcpu, exit_reason)) + return nested_vmx_reflect_vmexit(vcpu, exit_reason); + } if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) { dump_vmcs(); @@ -6237,15 +6220,13 @@ static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); /* if exit due to PF check for async PF */ - if (is_page_fault(vmx->exit_intr_info)) + if (is_page_fault(vmx->exit_intr_info)) { vmx->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason(); - /* Handle machine checks before interrupts are enabled */ - if (is_machine_check(vmx->exit_intr_info)) + } else if (is_machine_check(vmx->exit_intr_info)) { kvm_machine_check(); - /* We need to handle NMIs before interrupts are enabled */ - if (is_nmi(vmx->exit_intr_info)) { + } else if (is_nmi(vmx->exit_intr_info)) { kvm_before_interrupt(&vmx->vcpu); asm("int $2"); kvm_after_interrupt(&vmx->vcpu); @@ -6331,11 +6312,6 @@ static bool vmx_has_emulated_msr(int index) } } -static bool vmx_pt_supported(void) -{ - return pt_mode == PT_MODE_HOST_GUEST; -} - static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) { u32 exit_intr_info; @@ -6581,7 +6557,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) pt_guest_enter(vmx); - atomic_switch_perf_msrs(vmx); + if (vcpu_to_pmu(vcpu)->version) + atomic_switch_perf_msrs(vmx); atomic_switch_umwait_control_msr(vmx); if (enable_preemption_timer) @@ -6698,20 +6675,6 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) vmx_complete_interrupts(vmx); } -static struct kvm *vmx_vm_alloc(void) -{ - struct kvm_vmx *kvm_vmx = __vmalloc(sizeof(struct kvm_vmx), - GFP_KERNEL_ACCOUNT | __GFP_ZERO, - PAGE_KERNEL); - return &kvm_vmx->kvm; -} - -static void vmx_vm_free(struct kvm *kvm) -{ - kfree(kvm->arch.hyperv.hv_pa_pg); - vfree(to_kvm_vmx(kvm)); -} - static void vmx_free_vcpu(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -6914,17 +6877,24 @@ static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) u8 cache; u64 ipat = 0; - /* For VT-d and EPT combination - * 1. MMIO: always map as UC - * 2. EPT with VT-d: - * a. VT-d without snooping control feature: can't guarantee the - * result, try to trust guest. - * b. VT-d with snooping control feature: snooping control feature of - * VT-d engine can guarantee the cache correctness. Just set it - * to WB to keep consistent with host. So the same as item 3. - * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep - * consistent with host MTRR + /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in + * memory aliases with conflicting memory types and sometimes MCEs. + * We have to be careful as to what are honored and when. + * + * For MMIO, guest CD/MTRR are ignored. The EPT memory type is set to + * UC. The effective memory type is UC or WC depending on guest PAT. + * This was historically the source of MCEs and we want to be + * conservative. + * + * When there is no need to deal with noncoherent DMA (e.g., no VT-d + * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored. The + * EPT memory type is set to WB. The effective memory type is forced + * WB. + * + * Otherwise, we trust guest. Guest CD/MTRR/PAT are all honored. The + * EPT memory type is used to emulate guest CD/MTRR. */ + if (is_mmio) { cache = MTRR_TYPE_UNCACHABLE; goto exit; @@ -6951,15 +6921,6 @@ static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat; } -static int vmx_get_lpage_level(void) -{ - if (enable_ept && !cpu_has_vmx_ept_1g_page()) - return PT_DIRECTORY_LEVEL; - else - /* For shadow and EPT supported 1GB page */ - return PT_PDPE_LEVEL; -} - static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx) { /* @@ -7150,10 +7111,37 @@ static void vmx_cpuid_update(struct kvm_vcpu *vcpu) } } -static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) +static __init void vmx_set_cpu_caps(void) { - if (func == 1 && nested) - entry->ecx |= feature_bit(VMX); + kvm_set_cpu_caps(); + + /* CPUID 0x1 */ + if (nested) + kvm_cpu_cap_set(X86_FEATURE_VMX); + + /* CPUID 0x7 */ + if (kvm_mpx_supported()) + kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); + if (cpu_has_vmx_invpcid()) + kvm_cpu_cap_check_and_set(X86_FEATURE_INVPCID); + if (vmx_pt_mode_is_host_guest()) + kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); + + /* PKU is not yet implemented for shadow paging. */ + if (enable_ept && boot_cpu_has(X86_FEATURE_OSPKE)) + kvm_cpu_cap_check_and_set(X86_FEATURE_PKU); + + if (vmx_umip_emulated()) + kvm_cpu_cap_set(X86_FEATURE_UMIP); + + /* CPUID 0xD.1 */ + supported_xss = 0; + if (!vmx_xsaves_supported()) + kvm_cpu_cap_clear(X86_FEATURE_XSAVES); + + /* CPUID 0x80000001 */ + if (!cpu_has_vmx_rdtscp()) + kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); } static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu) @@ -7197,10 +7185,10 @@ static int vmx_check_intercept_io(struct kvm_vcpu *vcpu, static int vmx_check_intercept(struct kvm_vcpu *vcpu, struct x86_instruction_info *info, - enum x86_intercept_stage stage) + enum x86_intercept_stage stage, + struct x86_exception *exception) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; switch (info->intercept) { /* @@ -7209,8 +7197,8 @@ static int vmx_check_intercept(struct kvm_vcpu *vcpu, */ case x86_intercept_rdtscp: if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDTSCP)) { - ctxt->exception.vector = UD_VECTOR; - ctxt->exception.error_code_valid = false; + exception->vector = UD_VECTOR; + exception->error_code_valid = false; return X86EMUL_PROPAGATE_FAULT; } break; @@ -7321,7 +7309,8 @@ static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu) static void vmx_slot_enable_log_dirty(struct kvm *kvm, struct kvm_memory_slot *slot) { - kvm_mmu_slot_leaf_clear_dirty(kvm, slot); + if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) + kvm_mmu_slot_leaf_clear_dirty(kvm, slot); kvm_mmu_slot_largepage_remove_write_access(kvm, slot); } @@ -7504,7 +7493,7 @@ static void pi_post_block(struct kvm_vcpu *vcpu) static void vmx_post_block(struct kvm_vcpu *vcpu) { - if (kvm_x86_ops->set_hv_timer) + if (kvm_x86_ops.set_hv_timer) kvm_lapic_switch_to_hv_timer(vcpu); pi_post_block(vcpu); @@ -7671,171 +7660,7 @@ static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu) return to_vmx(vcpu)->nested.vmxon; } -static __init int hardware_setup(void) -{ - unsigned long host_bndcfgs; - struct desc_ptr dt; - int r, i; - - rdmsrl_safe(MSR_EFER, &host_efer); - - store_idt(&dt); - host_idt_base = dt.address; - - for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) - kvm_define_shared_msr(i, vmx_msr_index[i]); - - if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) - return -EIO; - - if (boot_cpu_has(X86_FEATURE_NX)) - kvm_enable_efer_bits(EFER_NX); - - if (boot_cpu_has(X86_FEATURE_MPX)) { - rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs); - WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost"); - } - - if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() || - !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global())) - enable_vpid = 0; - - if (!cpu_has_vmx_ept() || - !cpu_has_vmx_ept_4levels() || - !cpu_has_vmx_ept_mt_wb() || - !cpu_has_vmx_invept_global()) - enable_ept = 0; - - if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) - enable_ept_ad_bits = 0; - - if (!cpu_has_vmx_unrestricted_guest() || !enable_ept) - enable_unrestricted_guest = 0; - - if (!cpu_has_vmx_flexpriority()) - flexpriority_enabled = 0; - - if (!cpu_has_virtual_nmis()) - enable_vnmi = 0; - - /* - * set_apic_access_page_addr() is used to reload apic access - * page upon invalidation. No need to do anything if not - * using the APIC_ACCESS_ADDR VMCS field. - */ - if (!flexpriority_enabled) - kvm_x86_ops->set_apic_access_page_addr = NULL; - - if (!cpu_has_vmx_tpr_shadow()) - kvm_x86_ops->update_cr8_intercept = NULL; - - if (enable_ept && !cpu_has_vmx_ept_2m_page()) - kvm_disable_largepages(); - -#if IS_ENABLED(CONFIG_HYPERV) - if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH - && enable_ept) { - kvm_x86_ops->tlb_remote_flush = hv_remote_flush_tlb; - kvm_x86_ops->tlb_remote_flush_with_range = - hv_remote_flush_tlb_with_range; - } -#endif - - if (!cpu_has_vmx_ple()) { - ple_gap = 0; - ple_window = 0; - ple_window_grow = 0; - ple_window_max = 0; - ple_window_shrink = 0; - } - - if (!cpu_has_vmx_apicv()) { - enable_apicv = 0; - kvm_x86_ops->sync_pir_to_irr = NULL; - } - - if (cpu_has_vmx_tsc_scaling()) { - kvm_has_tsc_control = true; - kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; - kvm_tsc_scaling_ratio_frac_bits = 48; - } - - set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ - - if (enable_ept) - vmx_enable_tdp(); - else - kvm_disable_tdp(); - - /* - * Only enable PML when hardware supports PML feature, and both EPT - * and EPT A/D bit features are enabled -- PML depends on them to work. - */ - if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) - enable_pml = 0; - - if (!enable_pml) { - kvm_x86_ops->slot_enable_log_dirty = NULL; - kvm_x86_ops->slot_disable_log_dirty = NULL; - kvm_x86_ops->flush_log_dirty = NULL; - kvm_x86_ops->enable_log_dirty_pt_masked = NULL; - } - - if (!cpu_has_vmx_preemption_timer()) - enable_preemption_timer = false; - - if (enable_preemption_timer) { - u64 use_timer_freq = 5000ULL * 1000 * 1000; - u64 vmx_msr; - - rdmsrl(MSR_IA32_VMX_MISC, vmx_msr); - cpu_preemption_timer_multi = - vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; - - if (tsc_khz) - use_timer_freq = (u64)tsc_khz * 1000; - use_timer_freq >>= cpu_preemption_timer_multi; - - /* - * KVM "disables" the preemption timer by setting it to its max - * value. Don't use the timer if it might cause spurious exits - * at a rate faster than 0.1 Hz (of uninterrupted guest time). - */ - if (use_timer_freq > 0xffffffffu / 10) - enable_preemption_timer = false; - } - - if (!enable_preemption_timer) { - kvm_x86_ops->set_hv_timer = NULL; - kvm_x86_ops->cancel_hv_timer = NULL; - kvm_x86_ops->request_immediate_exit = __kvm_request_immediate_exit; - } - - kvm_set_posted_intr_wakeup_handler(wakeup_handler); - - kvm_mce_cap_supported |= MCG_LMCE_P; - - if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST) - return -EINVAL; - if (!enable_ept || !cpu_has_vmx_intel_pt()) - pt_mode = PT_MODE_SYSTEM; - - if (nested) { - nested_vmx_setup_ctls_msrs(&vmcs_config.nested, - vmx_capability.ept); - - r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); - if (r) - return r; - } - - r = alloc_kvm_area(); - if (r) - nested_vmx_hardware_unsetup(); - return r; -} - -static __exit void hardware_unsetup(void) +static void hardware_unsetup(void) { if (nested) nested_vmx_hardware_unsetup(); @@ -7851,20 +7676,16 @@ static bool vmx_check_apicv_inhibit_reasons(ulong bit) return supported & BIT(bit); } -static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { - .cpu_has_kvm_support = cpu_has_kvm_support, - .disabled_by_bios = vmx_disabled_by_bios, - .hardware_setup = hardware_setup, +static struct kvm_x86_ops vmx_x86_ops __initdata = { .hardware_unsetup = hardware_unsetup, - .check_processor_compatibility = vmx_check_processor_compat, + .hardware_enable = hardware_enable, .hardware_disable = hardware_disable, .cpu_has_accelerated_tpr = report_flexpriority, .has_emulated_msr = vmx_has_emulated_msr, + .vm_size = sizeof(struct kvm_vmx), .vm_init = vmx_vm_init, - .vm_alloc = vmx_vm_alloc, - .vm_free = vmx_vm_free, .vcpu_create = vmx_create_vcpu, .vcpu_free = vmx_free_vcpu, @@ -7886,7 +7707,6 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits, .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits, .set_cr0 = vmx_set_cr0, - .set_cr3 = vmx_set_cr3, .set_cr4 = vmx_set_cr4, .set_efer = vmx_set_efer, .get_idt = vmx_get_idt, @@ -7942,29 +7762,17 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { .get_exit_info = vmx_get_exit_info, - .get_lpage_level = vmx_get_lpage_level, - .cpuid_update = vmx_cpuid_update, - .rdtscp_supported = vmx_rdtscp_supported, - .invpcid_supported = vmx_invpcid_supported, - - .set_supported_cpuid = vmx_set_supported_cpuid, - .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, .read_l1_tsc_offset = vmx_read_l1_tsc_offset, .write_l1_tsc_offset = vmx_write_l1_tsc_offset, - .set_tdp_cr3 = vmx_set_cr3, + .load_mmu_pgd = vmx_load_mmu_pgd, .check_intercept = vmx_check_intercept, .handle_exit_irqoff = vmx_handle_exit_irqoff, - .mpx_supported = vmx_mpx_supported, - .xsaves_supported = vmx_xsaves_supported, - .umip_emulated = vmx_umip_emulated, - .pt_supported = vmx_pt_supported, - .pku_supported = vmx_pku_supported, .request_immediate_exit = vmx_request_immediate_exit, @@ -8005,6 +7813,189 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { .apic_init_signal_blocked = vmx_apic_init_signal_blocked, }; +static __init int hardware_setup(void) +{ + unsigned long host_bndcfgs; + struct desc_ptr dt; + int r, i, ept_lpage_level; + + store_idt(&dt); + host_idt_base = dt.address; + + for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) + kvm_define_shared_msr(i, vmx_msr_index[i]); + + if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) + return -EIO; + + if (boot_cpu_has(X86_FEATURE_NX)) + kvm_enable_efer_bits(EFER_NX); + + if (boot_cpu_has(X86_FEATURE_MPX)) { + rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs); + WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost"); + } + + if (!cpu_has_vmx_mpx()) + supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | + XFEATURE_MASK_BNDCSR); + + if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() || + !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global())) + enable_vpid = 0; + + if (!cpu_has_vmx_ept() || + !cpu_has_vmx_ept_4levels() || + !cpu_has_vmx_ept_mt_wb() || + !cpu_has_vmx_invept_global()) + enable_ept = 0; + + if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) + enable_ept_ad_bits = 0; + + if (!cpu_has_vmx_unrestricted_guest() || !enable_ept) + enable_unrestricted_guest = 0; + + if (!cpu_has_vmx_flexpriority()) + flexpriority_enabled = 0; + + if (!cpu_has_virtual_nmis()) + enable_vnmi = 0; + + /* + * set_apic_access_page_addr() is used to reload apic access + * page upon invalidation. No need to do anything if not + * using the APIC_ACCESS_ADDR VMCS field. + */ + if (!flexpriority_enabled) + vmx_x86_ops.set_apic_access_page_addr = NULL; + + if (!cpu_has_vmx_tpr_shadow()) + vmx_x86_ops.update_cr8_intercept = NULL; + +#if IS_ENABLED(CONFIG_HYPERV) + if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH + && enable_ept) { + vmx_x86_ops.tlb_remote_flush = hv_remote_flush_tlb; + vmx_x86_ops.tlb_remote_flush_with_range = + hv_remote_flush_tlb_with_range; + } +#endif + + if (!cpu_has_vmx_ple()) { + ple_gap = 0; + ple_window = 0; + ple_window_grow = 0; + ple_window_max = 0; + ple_window_shrink = 0; + } + + if (!cpu_has_vmx_apicv()) { + enable_apicv = 0; + vmx_x86_ops.sync_pir_to_irr = NULL; + } + + if (cpu_has_vmx_tsc_scaling()) { + kvm_has_tsc_control = true; + kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; + kvm_tsc_scaling_ratio_frac_bits = 48; + } + + set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ + + if (enable_ept) + vmx_enable_tdp(); + + if (!enable_ept) + ept_lpage_level = 0; + else if (cpu_has_vmx_ept_1g_page()) + ept_lpage_level = PT_PDPE_LEVEL; + else if (cpu_has_vmx_ept_2m_page()) + ept_lpage_level = PT_DIRECTORY_LEVEL; + else + ept_lpage_level = PT_PAGE_TABLE_LEVEL; + kvm_configure_mmu(enable_ept, ept_lpage_level); + + /* + * Only enable PML when hardware supports PML feature, and both EPT + * and EPT A/D bit features are enabled -- PML depends on them to work. + */ + if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) + enable_pml = 0; + + if (!enable_pml) { + vmx_x86_ops.slot_enable_log_dirty = NULL; + vmx_x86_ops.slot_disable_log_dirty = NULL; + vmx_x86_ops.flush_log_dirty = NULL; + vmx_x86_ops.enable_log_dirty_pt_masked = NULL; + } + + if (!cpu_has_vmx_preemption_timer()) + enable_preemption_timer = false; + + if (enable_preemption_timer) { + u64 use_timer_freq = 5000ULL * 1000 * 1000; + u64 vmx_msr; + + rdmsrl(MSR_IA32_VMX_MISC, vmx_msr); + cpu_preemption_timer_multi = + vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; + + if (tsc_khz) + use_timer_freq = (u64)tsc_khz * 1000; + use_timer_freq >>= cpu_preemption_timer_multi; + + /* + * KVM "disables" the preemption timer by setting it to its max + * value. Don't use the timer if it might cause spurious exits + * at a rate faster than 0.1 Hz (of uninterrupted guest time). + */ + if (use_timer_freq > 0xffffffffu / 10) + enable_preemption_timer = false; + } + + if (!enable_preemption_timer) { + vmx_x86_ops.set_hv_timer = NULL; + vmx_x86_ops.cancel_hv_timer = NULL; + vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit; + } + + kvm_set_posted_intr_wakeup_handler(wakeup_handler); + + kvm_mce_cap_supported |= MCG_LMCE_P; + + if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST) + return -EINVAL; + if (!enable_ept || !cpu_has_vmx_intel_pt()) + pt_mode = PT_MODE_SYSTEM; + + if (nested) { + nested_vmx_setup_ctls_msrs(&vmcs_config.nested, + vmx_capability.ept); + + r = nested_vmx_hardware_setup(&vmx_x86_ops, + kvm_vmx_exit_handlers); + if (r) + return r; + } + + vmx_set_cpu_caps(); + + r = alloc_kvm_area(); + if (r) + nested_vmx_hardware_unsetup(); + return r; +} + +static struct kvm_x86_init_ops vmx_init_ops __initdata = { + .cpu_has_kvm_support = cpu_has_kvm_support, + .disabled_by_bios = vmx_disabled_by_bios, + .check_processor_compatibility = vmx_check_processor_compat, + .hardware_setup = hardware_setup, + + .runtime_ops = &vmx_x86_ops, +}; + static void vmx_cleanup_l1d_flush(void) { if (vmx_l1d_flush_pages) { @@ -8089,7 +8080,7 @@ static int __init vmx_init(void) } #endif - r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), + r = kvm_init(&vmx_init_ops, sizeof(struct vcpu_vmx), __alignof__(struct vcpu_vmx), THIS_MODULE); if (r) return r; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 0695ea177e22..aab9df55336e 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -12,7 +12,6 @@ #include "vmcs.h" extern const u32 vmx_msr_index[]; -extern u64 host_efer; #define MSR_TYPE_R 1 #define MSR_TYPE_W 2 @@ -333,9 +332,9 @@ u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu); void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask); void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer); void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); -void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); void set_cr4_guest_host_mask(struct vcpu_vmx *vmx); +void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long cr3); void ept_save_pdptrs(struct kvm_vcpu *vcpu); void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); @@ -450,7 +449,7 @@ static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx) static inline u32 vmx_vmentry_ctrl(void) { u32 vmentry_ctrl = vmcs_config.vmentry_ctrl; - if (pt_mode == PT_MODE_SYSTEM) + if (vmx_pt_mode_is_system()) vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | VM_ENTRY_LOAD_IA32_RTIT_CTL); /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ @@ -461,7 +460,7 @@ static inline u32 vmx_vmentry_ctrl(void) static inline u32 vmx_vmexit_ctrl(void) { u32 vmexit_ctrl = vmcs_config.vmexit_ctrl; - if (pt_mode == PT_MODE_SYSTEM) + if (vmx_pt_mode_is_system()) vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | VM_EXIT_CLEAR_IA32_RTIT_CTL); /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ @@ -491,7 +490,6 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags); void free_vmcs(struct vmcs *vmcs); int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs); void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs); -void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs); void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs); static inline struct vmcs *alloc_vmcs(bool shadow) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index bf8564d73fc3..b8124b562dea 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -22,6 +22,7 @@ #include "i8254.h" #include "tss.h" #include "kvm_cache_regs.h" +#include "kvm_emulate.h" #include "x86.h" #include "cpuid.h" #include "pmu.h" @@ -81,7 +82,7 @@ u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P; EXPORT_SYMBOL_GPL(kvm_mce_cap_supported); #define emul_to_vcpu(ctxt) \ - container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt) + ((struct kvm_vcpu *)(ctxt)->vcpu) /* EFER defaults: * - enable syscall per default because its emulated by KVM @@ -109,7 +110,7 @@ static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); static void store_regs(struct kvm_vcpu *vcpu); static int sync_regs(struct kvm_vcpu *vcpu); -struct kvm_x86_ops *kvm_x86_ops __read_mostly; +struct kvm_x86_ops kvm_x86_ops __read_mostly; EXPORT_SYMBOL_GPL(kvm_x86_ops); static bool __read_mostly ignore_msrs = 0; @@ -180,7 +181,17 @@ struct kvm_shared_msrs { static struct kvm_shared_msrs_global __read_mostly shared_msrs_global; static struct kvm_shared_msrs __percpu *shared_msrs; +#define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ + | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ + | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ + | XFEATURE_MASK_PKRU) + +u64 __read_mostly host_efer; +EXPORT_SYMBOL_GPL(host_efer); + static u64 __read_mostly host_xss; +u64 __read_mostly supported_xss; +EXPORT_SYMBOL_GPL(supported_xss); struct kvm_stats_debugfs_item debugfs_entries[] = { { "pf_fixed", VCPU_STAT(pf_fixed) }, @@ -226,10 +237,25 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { }; u64 __read_mostly host_xcr0; +u64 __read_mostly supported_xcr0; +EXPORT_SYMBOL_GPL(supported_xcr0); struct kmem_cache *x86_fpu_cache; EXPORT_SYMBOL_GPL(x86_fpu_cache); +static struct kmem_cache *x86_emulator_cache; + +static struct kmem_cache *kvm_alloc_emulator_cache(void) +{ + unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src); + unsigned int size = sizeof(struct x86_emulate_ctxt); + + return kmem_cache_create_usercopy("x86_emulator", size, + __alignof__(struct x86_emulate_ctxt), + SLAB_ACCOUNT, useroffset, + size - useroffset, NULL); +} + static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt); static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) @@ -350,6 +376,7 @@ int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info) } kvm_lapic_set_base(vcpu, msr_info->data); + kvm_recalculate_apic_map(vcpu->kvm); return 0; } EXPORT_SYMBOL_GPL(kvm_set_apic_base); @@ -619,7 +646,7 @@ EXPORT_SYMBOL_GPL(kvm_requeue_exception_e); */ bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl) { - if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl) + if (kvm_x86_ops.get_cpl(vcpu) <= required_cpl) return true; kvm_queue_exception_e(vcpu, GP_VECTOR, 0); return false; @@ -760,7 +787,7 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (!is_pae(vcpu)) return 1; - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l); if (cs_l) return 1; } else @@ -773,7 +800,7 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)) return 1; - kvm_x86_ops->set_cr0(vcpu, cr0); + kvm_x86_ops.set_cr0(vcpu, cr0); if ((cr0 ^ old_cr0) & X86_CR0_PG) { kvm_clear_async_pf_completion_queue(vcpu); @@ -869,7 +896,7 @@ static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) { - if (kvm_x86_ops->get_cpl(vcpu) != 0 || + if (kvm_x86_ops.get_cpl(vcpu) != 0 || __kvm_set_xcr(vcpu, index, xcr)) { kvm_inject_gp(vcpu, 0); return 1; @@ -903,10 +930,10 @@ static u64 kvm_host_cr4_reserved_bits(struct cpuinfo_x86 *c) { u64 reserved_bits = __cr4_reserved_bits(cpu_has, c); - if (cpuid_ecx(0x7) & feature_bit(LA57)) + if (kvm_cpu_cap_has(X86_FEATURE_LA57)) reserved_bits &= ~X86_CR4_LA57; - if (kvm_x86_ops->umip_emulated()) + if (kvm_cpu_cap_has(X86_FEATURE_UMIP)) reserved_bits &= ~X86_CR4_UMIP; return reserved_bits; @@ -950,7 +977,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return 1; } - if (kvm_x86_ops->set_cr4(vcpu, cr4)) + if (kvm_x86_ops.set_cr4(vcpu, cr4)) return 1; if (((cr4 ^ old_cr4) & pdptr_bits) || @@ -1034,7 +1061,7 @@ static void kvm_update_dr0123(struct kvm_vcpu *vcpu) static void kvm_update_dr6(struct kvm_vcpu *vcpu) { if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) - kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6); + kvm_x86_ops.set_dr6(vcpu, vcpu->arch.dr6); } static void kvm_update_dr7(struct kvm_vcpu *vcpu) @@ -1045,7 +1072,7 @@ static void kvm_update_dr7(struct kvm_vcpu *vcpu) dr7 = vcpu->arch.guest_debug_dr7; else dr7 = vcpu->arch.dr7; - kvm_x86_ops->set_dr7(vcpu, dr7); + kvm_x86_ops.set_dr7(vcpu, dr7); vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; if (dr7 & DR7_BP_EN_MASK) vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; @@ -1115,7 +1142,7 @@ int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val) if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) *val = vcpu->arch.dr6; else - *val = kvm_x86_ops->get_dr6(vcpu); + *val = kvm_x86_ops.get_dr6(vcpu); break; case 5: /* fall through */ @@ -1350,7 +1377,7 @@ static int kvm_get_msr_feature(struct kvm_msr_entry *msr) rdmsrl_safe(msr->index, &msr->data); break; default: - if (kvm_x86_ops->get_msr_feature(msr)) + if (kvm_x86_ops.get_msr_feature(msr)) return 1; } return 0; @@ -1418,7 +1445,7 @@ static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) efer &= ~EFER_LMA; efer |= vcpu->arch.efer & EFER_LMA; - kvm_x86_ops->set_efer(vcpu, efer); + kvm_x86_ops.set_efer(vcpu, efer); /* Update reserved bits */ if ((efer ^ old_efer) & EFER_NX) @@ -1474,7 +1501,7 @@ static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, msr.index = index; msr.host_initiated = host_initiated; - return kvm_x86_ops->set_msr(vcpu, &msr); + return kvm_x86_ops.set_msr(vcpu, &msr); } /* @@ -1492,7 +1519,7 @@ int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, msr.index = index; msr.host_initiated = host_initiated; - ret = kvm_x86_ops->get_msr(vcpu, &msr); + ret = kvm_x86_ops.get_msr(vcpu, &msr); if (!ret) *data = msr.data; return ret; @@ -1561,8 +1588,12 @@ static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data ((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) && ((data & APIC_MODE_MASK) == APIC_DM_FIXED)) { + data &= ~(1 << 12); + kvm_apic_send_ipi(vcpu->arch.apic, (u32)data, (u32)(data >> 32)); kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32)); - return kvm_lapic_reg_write(vcpu->arch.apic, APIC_ICR, (u32)data); + kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR, (u32)data); + trace_kvm_apic_write(APIC_ICR, (u32)data); + return 0; } return 1; @@ -1571,11 +1602,12 @@ static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data enum exit_fastpath_completion handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu) { u32 msr = kvm_rcx_read(vcpu); - u64 data = kvm_read_edx_eax(vcpu); + u64 data; int ret = 0; switch (msr) { case APIC_BASE_MSR + (APIC_ICR >> 4): + data = kvm_read_edx_eax(vcpu); ret = handle_fastpath_set_x2apic_icr_irqoff(vcpu, data); break; default: @@ -1876,7 +1908,7 @@ static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset) { - u64 curr_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu); + u64 curr_offset = kvm_x86_ops.read_l1_tsc_offset(vcpu); vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset; } @@ -1918,7 +1950,7 @@ static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) { - u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu); + u64 tsc_offset = kvm_x86_ops.read_l1_tsc_offset(vcpu); return tsc_offset + kvm_scale_tsc(vcpu, host_tsc); } @@ -1926,7 +1958,7 @@ EXPORT_SYMBOL_GPL(kvm_read_l1_tsc); static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) { - vcpu->arch.tsc_offset = kvm_x86_ops->write_l1_tsc_offset(vcpu, offset); + vcpu->arch.tsc_offset = kvm_x86_ops.write_l1_tsc_offset(vcpu, offset); } static inline bool kvm_check_tsc_unstable(void) @@ -2050,7 +2082,7 @@ EXPORT_SYMBOL_GPL(kvm_write_tsc); static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment) { - u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu); + u64 tsc_offset = kvm_x86_ops.read_l1_tsc_offset(vcpu); kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment); } @@ -2525,7 +2557,7 @@ static void kvmclock_sync_fn(struct work_struct *work) static bool can_set_mci_status(struct kvm_vcpu *vcpu) { /* McStatusWrEn enabled? */ - if (guest_cpuid_is_amd(vcpu)) + if (guest_cpuid_is_amd_or_hygon(vcpu)) return !!(vcpu->arch.msr_hwcr & BIT_ULL(18)); return false; @@ -2647,7 +2679,7 @@ static void kvmclock_reset(struct kvm_vcpu *vcpu) static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa) { ++vcpu->stat.tlb_flush; - kvm_x86_ops->tlb_flush(vcpu, invalidate_gpa); + kvm_x86_ops.tlb_flush(vcpu, invalidate_gpa); } static void record_steal_time(struct kvm_vcpu *vcpu) @@ -2800,12 +2832,11 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) return 1; /* - * We do support PT if kvm_x86_ops->pt_supported(), but we do - * not support IA32_XSS[bit 8]. Guests will have to use - * RDMSR/WRMSR rather than XSAVES/XRSTORS to save/restore PT - * MSRs. + * KVM supports exposing PT to the guest, but does not support + * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than + * XSAVES/XRSTORS to save/restore PT MSRs. */ - if (data != 0) + if (data & ~supported_xss) return 1; vcpu->arch.ia32_xss = data; break; @@ -3079,7 +3110,6 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff: return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data); - break; case MSR_IA32_TSCDEADLINE: msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu); break; @@ -3162,7 +3192,6 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return kvm_hv_get_msr_common(vcpu, msr_info->index, &msr_info->data, msr_info->host_initiated); - break; case MSR_IA32_BBL_CR_CTL3: /* This legacy MSR exists but isn't fully documented in current * silicon. It is however accessed by winxp in very narrow @@ -3367,10 +3396,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) * fringe case that is not enabled except via specific settings * of the module parameters. */ - r = kvm_x86_ops->has_emulated_msr(MSR_IA32_SMBASE); + r = kvm_x86_ops.has_emulated_msr(MSR_IA32_SMBASE); break; case KVM_CAP_VAPIC: - r = !kvm_x86_ops->cpu_has_accelerated_tpr(); + r = !kvm_x86_ops.cpu_has_accelerated_tpr(); break; case KVM_CAP_NR_VCPUS: r = KVM_SOFT_MAX_VCPUS; @@ -3397,14 +3426,14 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = KVM_X2APIC_API_VALID_FLAGS; break; case KVM_CAP_NESTED_STATE: - r = kvm_x86_ops->get_nested_state ? - kvm_x86_ops->get_nested_state(NULL, NULL, 0) : 0; + r = kvm_x86_ops.get_nested_state ? + kvm_x86_ops.get_nested_state(NULL, NULL, 0) : 0; break; case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: - r = kvm_x86_ops->enable_direct_tlbflush != NULL; + r = kvm_x86_ops.enable_direct_tlbflush != NULL; break; case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: - r = kvm_x86_ops->nested_enable_evmcs != NULL; + r = kvm_x86_ops.nested_enable_evmcs != NULL; break; default: break; @@ -3466,7 +3495,7 @@ long kvm_arch_dev_ioctl(struct file *filp, r = 0; break; } - case KVM_X86_GET_MCE_CAP_SUPPORTED: { + case KVM_X86_GET_MCE_CAP_SUPPORTED: r = -EFAULT; if (copy_to_user(argp, &kvm_mce_cap_supported, sizeof(kvm_mce_cap_supported))) @@ -3498,9 +3527,9 @@ long kvm_arch_dev_ioctl(struct file *filp, case KVM_GET_MSRS: r = msr_io(NULL, argp, do_get_msr_feature, 1); break; - } default: r = -EINVAL; + break; } out: return r; @@ -3520,14 +3549,14 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { /* Address WBINVD may be executed by guest */ if (need_emulate_wbinvd(vcpu)) { - if (kvm_x86_ops->has_wbinvd_exit()) + if (kvm_x86_ops.has_wbinvd_exit()) cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); else if (vcpu->cpu != -1 && vcpu->cpu != cpu) smp_call_function_single(vcpu->cpu, wbinvd_ipi, NULL, 1); } - kvm_x86_ops->vcpu_load(vcpu, cpu); + kvm_x86_ops.vcpu_load(vcpu, cpu); /* Apply any externally detected TSC adjustments (due to suspend) */ if (unlikely(vcpu->arch.tsc_offset_adjustment)) { @@ -3594,7 +3623,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) int idx; if (vcpu->preempted) - vcpu->arch.preempted_in_kernel = !kvm_x86_ops->get_cpl(vcpu); + vcpu->arch.preempted_in_kernel = !kvm_x86_ops.get_cpl(vcpu); /* * Disable page faults because we're in atomic context here. @@ -3613,7 +3642,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_steal_time_set_preempted(vcpu); srcu_read_unlock(&vcpu->kvm->srcu, idx); pagefault_enable(); - kvm_x86_ops->vcpu_put(vcpu); + kvm_x86_ops.vcpu_put(vcpu); vcpu->arch.last_host_tsc = rdtsc(); /* * If userspace has set any breakpoints or watchpoints, dr6 is restored @@ -3627,7 +3656,7 @@ static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) { if (vcpu->arch.apicv_active) - kvm_x86_ops->sync_pir_to_irr(vcpu); + kvm_x86_ops.sync_pir_to_irr(vcpu); return kvm_apic_get_state(vcpu, s); } @@ -3735,7 +3764,7 @@ static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, for (bank = 0; bank < bank_num; bank++) vcpu->arch.mce_banks[bank*4] = ~(u64)0; - kvm_x86_ops->setup_mce(vcpu); + kvm_x86_ops.setup_mce(vcpu); out: return r; } @@ -3839,11 +3868,11 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft; events->interrupt.nr = vcpu->arch.interrupt.nr; events->interrupt.soft = 0; - events->interrupt.shadow = kvm_x86_ops->get_interrupt_shadow(vcpu); + events->interrupt.shadow = kvm_x86_ops.get_interrupt_shadow(vcpu); events->nmi.injected = vcpu->arch.nmi_injected; events->nmi.pending = vcpu->arch.nmi_pending != 0; - events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu); + events->nmi.masked = kvm_x86_ops.get_nmi_mask(vcpu); events->nmi.pad = 0; events->sipi_vector = 0; /* never valid when reporting to user space */ @@ -3910,13 +3939,13 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, vcpu->arch.interrupt.nr = events->interrupt.nr; vcpu->arch.interrupt.soft = events->interrupt.soft; if (events->flags & KVM_VCPUEVENT_VALID_SHADOW) - kvm_x86_ops->set_interrupt_shadow(vcpu, + kvm_x86_ops.set_interrupt_shadow(vcpu, events->interrupt.shadow); vcpu->arch.nmi_injected = events->nmi.injected; if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING) vcpu->arch.nmi_pending = events->nmi.pending; - kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked); + kvm_x86_ops.set_nmi_mask(vcpu, events->nmi.masked); if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR && lapic_in_kernel(vcpu)) @@ -4103,8 +4132,7 @@ static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, * CPUID leaf 0xD, index 0, EDX:EAX. This is for compatibility * with old userspace. */ - if (xstate_bv & ~kvm_supported_xcr0() || - mxcsr & ~mxcsr_feature_mask) + if (xstate_bv & ~supported_xcr0 || mxcsr & ~mxcsr_feature_mask) return -EINVAL; load_xsave(vcpu, (u8 *)guest_xsave->region); } else { @@ -4191,9 +4219,9 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, return kvm_hv_activate_synic(vcpu, cap->cap == KVM_CAP_HYPERV_SYNIC2); case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: - if (!kvm_x86_ops->nested_enable_evmcs) + if (!kvm_x86_ops.nested_enable_evmcs) return -ENOTTY; - r = kvm_x86_ops->nested_enable_evmcs(vcpu, &vmcs_version); + r = kvm_x86_ops.nested_enable_evmcs(vcpu, &vmcs_version); if (!r) { user_ptr = (void __user *)(uintptr_t)cap->args[0]; if (copy_to_user(user_ptr, &vmcs_version, @@ -4202,10 +4230,10 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, } return r; case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: - if (!kvm_x86_ops->enable_direct_tlbflush) + if (!kvm_x86_ops.enable_direct_tlbflush) return -ENOTTY; - return kvm_x86_ops->enable_direct_tlbflush(vcpu); + return kvm_x86_ops.enable_direct_tlbflush(vcpu); default: return -EINVAL; @@ -4508,7 +4536,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, u32 user_data_size; r = -EINVAL; - if (!kvm_x86_ops->get_nested_state) + if (!kvm_x86_ops.get_nested_state) break; BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size)); @@ -4516,7 +4544,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, if (get_user(user_data_size, &user_kvm_nested_state->size)) break; - r = kvm_x86_ops->get_nested_state(vcpu, user_kvm_nested_state, + r = kvm_x86_ops.get_nested_state(vcpu, user_kvm_nested_state, user_data_size); if (r < 0) break; @@ -4538,7 +4566,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, int idx; r = -EINVAL; - if (!kvm_x86_ops->set_nested_state) + if (!kvm_x86_ops.set_nested_state) break; r = -EFAULT; @@ -4560,7 +4588,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; idx = srcu_read_lock(&vcpu->kvm->srcu); - r = kvm_x86_ops->set_nested_state(vcpu, user_kvm_nested_state, &kvm_state); + r = kvm_x86_ops.set_nested_state(vcpu, user_kvm_nested_state, &kvm_state); srcu_read_unlock(&vcpu->kvm->srcu, idx); break; } @@ -4604,14 +4632,14 @@ static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) if (addr > (unsigned int)(-3 * PAGE_SIZE)) return -EINVAL; - ret = kvm_x86_ops->set_tss_addr(kvm, addr); + ret = kvm_x86_ops.set_tss_addr(kvm, addr); return ret; } static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) { - return kvm_x86_ops->set_identity_map_addr(kvm, ident_addr); + return kvm_x86_ops.set_identity_map_addr(kvm, ident_addr); } static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, @@ -4763,77 +4791,13 @@ static int kvm_vm_ioctl_reinject(struct kvm *kvm, return 0; } -/** - * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot - * @kvm: kvm instance - * @log: slot id and address to which we copy the log - * - * Steps 1-4 below provide general overview of dirty page logging. See - * kvm_get_dirty_log_protect() function description for additional details. - * - * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we - * always flush the TLB (step 4) even if previous step failed and the dirty - * bitmap may be corrupt. Regardless of previous outcome the KVM logging API - * does not preclude user space subsequent dirty log read. Flushing TLB ensures - * writes will be marked dirty for next log read. - * - * 1. Take a snapshot of the bit and clear it if needed. - * 2. Write protect the corresponding page. - * 3. Copy the snapshot to the userspace. - * 4. Flush TLB's if needed. - */ -int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { - bool flush = false; - int r; - - mutex_lock(&kvm->slots_lock); - /* * Flush potentially hardware-cached dirty pages to dirty_bitmap. */ - if (kvm_x86_ops->flush_log_dirty) - kvm_x86_ops->flush_log_dirty(kvm); - - r = kvm_get_dirty_log_protect(kvm, log, &flush); - - /* - * All the TLBs can be flushed out of mmu lock, see the comments in - * kvm_mmu_slot_remove_write_access(). - */ - lockdep_assert_held(&kvm->slots_lock); - if (flush) - kvm_flush_remote_tlbs(kvm); - - mutex_unlock(&kvm->slots_lock); - return r; -} - -int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log) -{ - bool flush = false; - int r; - - mutex_lock(&kvm->slots_lock); - - /* - * Flush potentially hardware-cached dirty pages to dirty_bitmap. - */ - if (kvm_x86_ops->flush_log_dirty) - kvm_x86_ops->flush_log_dirty(kvm); - - r = kvm_clear_dirty_log_protect(kvm, log, &flush); - - /* - * All the TLBs can be flushed out of mmu lock, see the comments in - * kvm_mmu_slot_remove_write_access(). - */ - lockdep_assert_held(&kvm->slots_lock); - if (flush) - kvm_flush_remote_tlbs(kvm); - - mutex_unlock(&kvm->slots_lock); - return r; + if (kvm_x86_ops.flush_log_dirty) + kvm_x86_ops.flush_log_dirty(kvm); } int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, @@ -5186,8 +5150,8 @@ long kvm_arch_vm_ioctl(struct file *filp, } case KVM_MEMORY_ENCRYPT_OP: { r = -ENOTTY; - if (kvm_x86_ops->mem_enc_op) - r = kvm_x86_ops->mem_enc_op(kvm, argp); + if (kvm_x86_ops.mem_enc_op) + r = kvm_x86_ops.mem_enc_op(kvm, argp); break; } case KVM_MEMORY_ENCRYPT_REG_REGION: { @@ -5198,8 +5162,8 @@ long kvm_arch_vm_ioctl(struct file *filp, goto out; r = -ENOTTY; - if (kvm_x86_ops->mem_enc_reg_region) - r = kvm_x86_ops->mem_enc_reg_region(kvm, ®ion); + if (kvm_x86_ops.mem_enc_reg_region) + r = kvm_x86_ops.mem_enc_reg_region(kvm, ®ion); break; } case KVM_MEMORY_ENCRYPT_UNREG_REGION: { @@ -5210,8 +5174,8 @@ long kvm_arch_vm_ioctl(struct file *filp, goto out; r = -ENOTTY; - if (kvm_x86_ops->mem_enc_unreg_region) - r = kvm_x86_ops->mem_enc_unreg_region(kvm, ®ion); + if (kvm_x86_ops.mem_enc_unreg_region) + r = kvm_x86_ops.mem_enc_unreg_region(kvm, ®ion); break; } case KVM_HYPERV_EVENTFD: { @@ -5262,28 +5226,28 @@ static void kvm_init_msr_list(void) continue; break; case MSR_TSC_AUX: - if (!kvm_x86_ops->rdtscp_supported()) + if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) continue; break; case MSR_IA32_RTIT_CTL: case MSR_IA32_RTIT_STATUS: - if (!kvm_x86_ops->pt_supported()) + if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) continue; break; case MSR_IA32_RTIT_CR3_MATCH: - if (!kvm_x86_ops->pt_supported() || + if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering)) continue; break; case MSR_IA32_RTIT_OUTPUT_BASE: case MSR_IA32_RTIT_OUTPUT_MASK: - if (!kvm_x86_ops->pt_supported() || + if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || (!intel_pt_validate_hw_cap(PT_CAP_topa_output) && !intel_pt_validate_hw_cap(PT_CAP_single_range_output))) continue; break; case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: { - if (!kvm_x86_ops->pt_supported() || + if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >= intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2) continue; @@ -5306,7 +5270,7 @@ static void kvm_init_msr_list(void) } for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) { - if (!kvm_x86_ops->has_emulated_msr(emulated_msrs_all[i])) + if (!kvm_x86_ops.has_emulated_msr(emulated_msrs_all[i])) continue; emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i]; @@ -5369,13 +5333,13 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v) static void kvm_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) { - kvm_x86_ops->set_segment(vcpu, var, seg); + kvm_x86_ops.set_segment(vcpu, var, seg); } void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) { - kvm_x86_ops->get_segment(vcpu, var, seg); + kvm_x86_ops.get_segment(vcpu, var, seg); } gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, @@ -5395,14 +5359,14 @@ gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception) { - u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; + u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); } gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception) { - u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; + u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_FETCH_MASK; return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); } @@ -5410,7 +5374,7 @@ gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception) { - u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; + u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_WRITE_MASK; return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); } @@ -5459,7 +5423,7 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, struct x86_exception *exception) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); - u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; + u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; unsigned offset; int ret; @@ -5484,7 +5448,7 @@ int kvm_read_guest_virt(struct kvm_vcpu *vcpu, gva_t addr, void *val, unsigned int bytes, struct x86_exception *exception) { - u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; + u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; /* * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED @@ -5505,7 +5469,7 @@ static int emulator_read_std(struct x86_emulate_ctxt *ctxt, struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); u32 access = 0; - if (!system && kvm_x86_ops->get_cpl(vcpu) == 3) + if (!system && kvm_x86_ops.get_cpl(vcpu) == 3) access |= PFERR_USER_MASK; return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception); @@ -5558,7 +5522,7 @@ static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *v struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); u32 access = PFERR_WRITE_MASK; - if (!system && kvm_x86_ops->get_cpl(vcpu) == 3) + if (!system && kvm_x86_ops.get_cpl(vcpu) == 3) access |= PFERR_USER_MASK; return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, @@ -5621,7 +5585,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, gpa_t *gpa, struct x86_exception *exception, bool write) { - u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0) + u32 access = ((kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0) | (write ? PFERR_WRITE_MASK : 0); /* @@ -5740,7 +5704,7 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, int handled, ret; bool write = ops->write; struct kvm_mmio_fragment *frag; - struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; + struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; /* * If the exit was due to a NPF we may already have a GPA. @@ -5749,10 +5713,9 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, * operation using rep will only have the initial GPA from the NPF * occurred. */ - if (vcpu->arch.gpa_available && - emulator_can_use_gpa(ctxt) && - (addr & ~PAGE_MASK) == (vcpu->arch.gpa_val & ~PAGE_MASK)) { - gpa = vcpu->arch.gpa_val; + if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) && + (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) { + gpa = ctxt->gpa_val; ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); } else { ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write); @@ -5972,11 +5935,9 @@ static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size, return 0; } -static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt, - int size, unsigned short port, void *val, - unsigned int count) +static int emulator_pio_in(struct kvm_vcpu *vcpu, int size, + unsigned short port, void *val, unsigned int count) { - struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); int ret; if (vcpu->arch.pio.count) @@ -5996,20 +5957,33 @@ static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt, return 0; } -static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, - int size, unsigned short port, - const void *val, unsigned int count) +static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt, + int size, unsigned short port, void *val, + unsigned int count) { - struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); + return emulator_pio_in(emul_to_vcpu(ctxt), size, port, val, count); +} + +static int emulator_pio_out(struct kvm_vcpu *vcpu, int size, + unsigned short port, const void *val, + unsigned int count) +{ memcpy(vcpu->arch.pio_data, val, size * count); trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data); return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false); } +static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, + int size, unsigned short port, + const void *val, unsigned int count) +{ + return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count); +} + static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) { - return kvm_x86_ops->get_segment_base(vcpu, seg); + return kvm_x86_ops.get_segment_base(vcpu, seg); } static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address) @@ -6022,7 +5996,7 @@ static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) if (!need_emulate_wbinvd(vcpu)) return X86EMUL_CONTINUE; - if (kvm_x86_ops->has_wbinvd_exit()) { + if (kvm_x86_ops.has_wbinvd_exit()) { int cpu = get_cpu(); cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); @@ -6127,27 +6101,27 @@ static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val) static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt) { - return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt)); + return kvm_x86_ops.get_cpl(emul_to_vcpu(ctxt)); } static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) { - kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt); + kvm_x86_ops.get_gdt(emul_to_vcpu(ctxt), dt); } static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) { - kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt); + kvm_x86_ops.get_idt(emul_to_vcpu(ctxt), dt); } static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) { - kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt); + kvm_x86_ops.set_gdt(emul_to_vcpu(ctxt), dt); } static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) { - kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt); + kvm_x86_ops.set_idt(emul_to_vcpu(ctxt), dt); } static unsigned long emulator_get_cached_segment_base( @@ -6269,13 +6243,15 @@ static int emulator_intercept(struct x86_emulate_ctxt *ctxt, struct x86_instruction_info *info, enum x86_intercept_stage stage) { - return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage); + return kvm_x86_ops.check_intercept(emul_to_vcpu(ctxt), info, stage, + &ctxt->exception); } static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt, - u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, bool check_limit) + u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, + bool exact_only) { - return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, check_limit); + return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only); } static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt) @@ -6305,7 +6281,7 @@ static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulon static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked) { - kvm_x86_ops->set_nmi_mask(emul_to_vcpu(ctxt), masked); + kvm_x86_ops.set_nmi_mask(emul_to_vcpu(ctxt), masked); } static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt) @@ -6321,7 +6297,7 @@ static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_fla static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt, const char *smstate) { - return kvm_x86_ops->pre_leave_smm(emul_to_vcpu(ctxt), smstate); + return kvm_x86_ops.pre_leave_smm(emul_to_vcpu(ctxt), smstate); } static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt) @@ -6383,7 +6359,7 @@ static const struct x86_emulate_ops emulate_ops = { static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) { - u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu); + u32 int_shadow = kvm_x86_ops.get_interrupt_shadow(vcpu); /* * an sti; sti; sequence only disable interrupts for the first * instruction. So, if the last instruction, be it emulated or @@ -6394,7 +6370,7 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) if (int_shadow & mask) mask = 0; if (unlikely(int_shadow || mask)) { - kvm_x86_ops->set_interrupt_shadow(vcpu, mask); + kvm_x86_ops.set_interrupt_shadow(vcpu, mask); if (!mask) kvm_make_request(KVM_REQ_EVENT, vcpu); } @@ -6402,7 +6378,7 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) static bool inject_emulated_exception(struct kvm_vcpu *vcpu) { - struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; + struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; if (ctxt->exception.vector == PF_VECTOR) return kvm_propagate_fault(vcpu, &ctxt->exception); @@ -6414,13 +6390,31 @@ static bool inject_emulated_exception(struct kvm_vcpu *vcpu) return false; } +static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu) +{ + struct x86_emulate_ctxt *ctxt; + + ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT); + if (!ctxt) { + pr_err("kvm: failed to allocate vcpu's emulator\n"); + return NULL; + } + + ctxt->vcpu = vcpu; + ctxt->ops = &emulate_ops; + vcpu->arch.emulate_ctxt = ctxt; + + return ctxt; +} + static void init_emulate_ctxt(struct kvm_vcpu *vcpu) { - struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; + struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; int cs_db, cs_l; - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + ctxt->gpa_available = false; ctxt->eflags = kvm_get_rflags(vcpu); ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; @@ -6440,7 +6434,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) { - struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; + struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; int ret; init_emulate_ctxt(vcpu); @@ -6479,7 +6473,7 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) kvm_queue_exception(vcpu, UD_VECTOR); - if (!is_guest_mode(vcpu) && kvm_x86_ops->get_cpl(vcpu) == 0) { + if (!is_guest_mode(vcpu) && kvm_x86_ops.get_cpl(vcpu) == 0) { vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; vcpu->run->internal.ndata = 0; @@ -6496,10 +6490,11 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, gpa_t gpa = cr2_or_gpa; kvm_pfn_t pfn; - if (!(emulation_type & EMULTYPE_ALLOW_RETRY)) + if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF)) return false; - if (WARN_ON_ONCE(is_guest_mode(vcpu))) + if (WARN_ON_ONCE(is_guest_mode(vcpu)) || + WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF))) return false; if (!vcpu->arch.mmu->direct_map) { @@ -6587,10 +6582,11 @@ static bool retry_instruction(struct x86_emulate_ctxt *ctxt, */ vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0; - if (!(emulation_type & EMULTYPE_ALLOW_RETRY)) + if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF)) return false; - if (WARN_ON_ONCE(is_guest_mode(vcpu))) + if (WARN_ON_ONCE(is_guest_mode(vcpu)) || + WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF))) return false; if (x86_page_table_writing_insn(ctxt)) @@ -6658,10 +6654,10 @@ static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu) int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) { - unsigned long rflags = kvm_x86_ops->get_rflags(vcpu); + unsigned long rflags = kvm_x86_ops.get_rflags(vcpu); int r; - r = kvm_x86_ops->skip_emulated_instruction(vcpu); + r = kvm_x86_ops.skip_emulated_instruction(vcpu); if (unlikely(!r)) return 0; @@ -6753,7 +6749,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, int emulation_type, void *insn, int insn_len) { int r; - struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; + struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; bool writeback = true; bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable; @@ -6843,8 +6839,19 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, } restart: - /* Save the faulting GPA (cr2) in the address field */ - ctxt->exception.address = cr2_or_gpa; + if (emulation_type & EMULTYPE_PF) { + /* Save the faulting GPA (cr2) in the address field */ + ctxt->exception.address = cr2_or_gpa; + + /* With shadow page tables, cr2 contains a GVA or nGPA. */ + if (vcpu->arch.mmu->direct_map) { + ctxt->gpa_available = true; + ctxt->gpa_val = cr2_or_gpa; + } + } else { + /* Sanitize the address out of an abundance of paranoia. */ + ctxt->exception.address = 0; + } r = x86_emulate_insn(ctxt); @@ -6885,7 +6892,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, r = 1; if (writeback) { - unsigned long rflags = kvm_x86_ops->get_rflags(vcpu); + unsigned long rflags = kvm_x86_ops.get_rflags(vcpu); toggle_interruptibility(vcpu, ctxt->interruptibility); vcpu->arch.emulate_regs_need_sync_to_vcpu = false; if (!ctxt->have_exception || @@ -6893,8 +6900,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, kvm_rip_write(vcpu, ctxt->eip); if (r && ctxt->tf) r = kvm_vcpu_do_singlestep(vcpu); - if (kvm_x86_ops->update_emulated_instruction) - kvm_x86_ops->update_emulated_instruction(vcpu); + if (kvm_x86_ops.update_emulated_instruction) + kvm_x86_ops.update_emulated_instruction(vcpu); __kvm_set_rflags(vcpu, ctxt->eflags); } @@ -6945,8 +6952,8 @@ static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port) { unsigned long val = kvm_rax_read(vcpu); - int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt, - size, port, &val, 1); + int ret = emulator_pio_out(vcpu, size, port, &val, 1); + if (ret) return ret; @@ -6982,11 +6989,10 @@ static int complete_fast_pio_in(struct kvm_vcpu *vcpu) val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0; /* - * Since vcpu->arch.pio.count == 1 let emulator_pio_in_emulated perform + * Since vcpu->arch.pio.count == 1 let emulator_pio_in perform * the copy and tracing */ - emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, vcpu->arch.pio.size, - vcpu->arch.pio.port, &val, 1); + emulator_pio_in(vcpu, vcpu->arch.pio.size, vcpu->arch.pio.port, &val, 1); kvm_rax_write(vcpu, val); return kvm_skip_emulated_instruction(vcpu); @@ -7001,8 +7007,7 @@ static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, /* For size less than 4 we merge, else we zero extend */ val = (size < 4) ? kvm_rax_read(vcpu) : 0; - ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port, - &val, 1); + ret = emulator_pio_in(vcpu, size, port, &val, 1); if (ret) { kvm_rax_write(vcpu, val); return ret; @@ -7225,7 +7230,7 @@ static int kvm_is_user_mode(void) int user_mode = 3; if (__this_cpu_read(current_vcpu)) - user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu)); + user_mode = kvm_x86_ops.get_cpl(__this_cpu_read(current_vcpu)); return user_mode != 0; } @@ -7302,10 +7307,10 @@ static struct notifier_block pvclock_gtod_notifier = { int kvm_arch_init(void *opaque) { + struct kvm_x86_init_ops *ops = opaque; int r; - struct kvm_x86_ops *ops = opaque; - if (kvm_x86_ops) { + if (kvm_x86_ops.hardware_enable) { printk(KERN_ERR "kvm: already loaded the other module\n"); r = -EEXIST; goto out; @@ -7342,18 +7347,22 @@ int kvm_arch_init(void *opaque) goto out; } + x86_emulator_cache = kvm_alloc_emulator_cache(); + if (!x86_emulator_cache) { + pr_err("kvm: failed to allocate cache for x86 emulator\n"); + goto out_free_x86_fpu_cache; + } + shared_msrs = alloc_percpu(struct kvm_shared_msrs); if (!shared_msrs) { printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n"); - goto out_free_x86_fpu_cache; + goto out_free_x86_emulator_cache; } r = kvm_mmu_module_init(); if (r) goto out_free_percpu; - kvm_x86_ops = ops; - kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, PT_DIRTY_MASK, PT64_NX_MASK, 0, PT_PRESENT_MASK, 0, sme_me_mask); @@ -7361,8 +7370,10 @@ int kvm_arch_init(void *opaque) perf_register_guest_info_callbacks(&kvm_guest_cbs); - if (boot_cpu_has(X86_FEATURE_XSAVE)) + if (boot_cpu_has(X86_FEATURE_XSAVE)) { host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); + supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0; + } kvm_lapic_init(); if (pi_inject_timer == -1) @@ -7378,6 +7389,8 @@ int kvm_arch_init(void *opaque) out_free_percpu: free_percpu(shared_msrs); +out_free_x86_emulator_cache: + kmem_cache_destroy(x86_emulator_cache); out_free_x86_fpu_cache: kmem_cache_destroy(x86_fpu_cache); out: @@ -7400,7 +7413,7 @@ void kvm_arch_exit(void) #ifdef CONFIG_X86_64 pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier); #endif - kvm_x86_ops = NULL; + kvm_x86_ops.hardware_enable = NULL; kvm_mmu_module_exit(); free_percpu(shared_msrs); kmem_cache_destroy(x86_fpu_cache); @@ -7538,7 +7551,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) a3 &= 0xFFFFFFFF; } - if (kvm_x86_ops->get_cpl(vcpu) != 0) { + if (kvm_x86_ops.get_cpl(vcpu) != 0) { ret = -KVM_EPERM; goto out; } @@ -7584,7 +7597,7 @@ static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt) char instruction[3]; unsigned long rip = kvm_rip_read(vcpu); - kvm_x86_ops->patch_hypercall(vcpu, instruction); + kvm_x86_ops.patch_hypercall(vcpu, instruction); return emulator_write_emulated(ctxt, rip, instruction, 3, &ctxt->exception); @@ -7613,7 +7626,7 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu) { int max_irr, tpr; - if (!kvm_x86_ops->update_cr8_intercept) + if (!kvm_x86_ops.update_cr8_intercept) return; if (!lapic_in_kernel(vcpu)) @@ -7632,17 +7645,17 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu) tpr = kvm_lapic_get_cr8(vcpu); - kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr); + kvm_x86_ops.update_cr8_intercept(vcpu, tpr, max_irr); } -static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win) +static int inject_pending_event(struct kvm_vcpu *vcpu) { int r; /* try to reinject previous events if any */ if (vcpu->arch.exception.injected) - kvm_x86_ops->queue_exception(vcpu); + kvm_x86_ops.queue_exception(vcpu); /* * Do not inject an NMI or interrupt if there is a pending * exception. Exceptions and interrupts are recognized at @@ -7659,9 +7672,9 @@ static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win) */ else if (!vcpu->arch.exception.pending) { if (vcpu->arch.nmi_injected) - kvm_x86_ops->set_nmi(vcpu); + kvm_x86_ops.set_nmi(vcpu); else if (vcpu->arch.interrupt.injected) - kvm_x86_ops->set_irq(vcpu); + kvm_x86_ops.set_irq(vcpu); } /* @@ -7670,8 +7683,8 @@ static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win) * from L2 to L1 due to pending L1 events which require exit * from L2 to L1. */ - if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) { - r = kvm_x86_ops->check_nested_events(vcpu, req_int_win); + if (is_guest_mode(vcpu) && kvm_x86_ops.check_nested_events) { + r = kvm_x86_ops.check_nested_events(vcpu); if (r != 0) return r; } @@ -7708,7 +7721,7 @@ static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win) } } - kvm_x86_ops->queue_exception(vcpu); + kvm_x86_ops.queue_exception(vcpu); } /* Don't consider new event if we re-injected an event */ @@ -7716,14 +7729,14 @@ static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win) return 0; if (vcpu->arch.smi_pending && !is_smm(vcpu) && - kvm_x86_ops->smi_allowed(vcpu)) { + kvm_x86_ops.smi_allowed(vcpu)) { vcpu->arch.smi_pending = false; ++vcpu->arch.smi_count; enter_smm(vcpu); - } else if (vcpu->arch.nmi_pending && kvm_x86_ops->nmi_allowed(vcpu)) { + } else if (vcpu->arch.nmi_pending && kvm_x86_ops.nmi_allowed(vcpu)) { --vcpu->arch.nmi_pending; vcpu->arch.nmi_injected = true; - kvm_x86_ops->set_nmi(vcpu); + kvm_x86_ops.set_nmi(vcpu); } else if (kvm_cpu_has_injectable_intr(vcpu)) { /* * Because interrupts can be injected asynchronously, we are @@ -7732,15 +7745,15 @@ static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win) * proposal and current concerns. Perhaps we should be setting * KVM_REQ_EVENT only on certain events and not unconditionally? */ - if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) { - r = kvm_x86_ops->check_nested_events(vcpu, req_int_win); + if (is_guest_mode(vcpu) && kvm_x86_ops.check_nested_events) { + r = kvm_x86_ops.check_nested_events(vcpu); if (r != 0) return r; } - if (kvm_x86_ops->interrupt_allowed(vcpu)) { + if (kvm_x86_ops.interrupt_allowed(vcpu)) { kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false); - kvm_x86_ops->set_irq(vcpu); + kvm_x86_ops.set_irq(vcpu); } } @@ -7756,7 +7769,7 @@ static void process_nmi(struct kvm_vcpu *vcpu) * If an NMI is already in progress, limit further NMIs to just one. * Otherwise, allow two (and we'll inject the first one immediately). */ - if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected) + if (kvm_x86_ops.get_nmi_mask(vcpu) || vcpu->arch.nmi_injected) limit = 1; vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0); @@ -7846,11 +7859,11 @@ static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf) put_smstate(u32, buf, 0x7f7c, seg.limit); put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg)); - kvm_x86_ops->get_gdt(vcpu, &dt); + kvm_x86_ops.get_gdt(vcpu, &dt); put_smstate(u32, buf, 0x7f74, dt.address); put_smstate(u32, buf, 0x7f70, dt.size); - kvm_x86_ops->get_idt(vcpu, &dt); + kvm_x86_ops.get_idt(vcpu, &dt); put_smstate(u32, buf, 0x7f58, dt.address); put_smstate(u32, buf, 0x7f54, dt.size); @@ -7900,7 +7913,7 @@ static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf) put_smstate(u32, buf, 0x7e94, seg.limit); put_smstate(u64, buf, 0x7e98, seg.base); - kvm_x86_ops->get_idt(vcpu, &dt); + kvm_x86_ops.get_idt(vcpu, &dt); put_smstate(u32, buf, 0x7e84, dt.size); put_smstate(u64, buf, 0x7e88, dt.address); @@ -7910,7 +7923,7 @@ static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf) put_smstate(u32, buf, 0x7e74, seg.limit); put_smstate(u64, buf, 0x7e78, seg.base); - kvm_x86_ops->get_gdt(vcpu, &dt); + kvm_x86_ops.get_gdt(vcpu, &dt); put_smstate(u32, buf, 0x7e64, dt.size); put_smstate(u64, buf, 0x7e68, dt.address); @@ -7940,28 +7953,28 @@ static void enter_smm(struct kvm_vcpu *vcpu) * vCPU state (e.g. leave guest mode) after we've saved the state into * the SMM state-save area. */ - kvm_x86_ops->pre_enter_smm(vcpu, buf); + kvm_x86_ops.pre_enter_smm(vcpu, buf); vcpu->arch.hflags |= HF_SMM_MASK; kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf)); - if (kvm_x86_ops->get_nmi_mask(vcpu)) + if (kvm_x86_ops.get_nmi_mask(vcpu)) vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK; else - kvm_x86_ops->set_nmi_mask(vcpu, true); + kvm_x86_ops.set_nmi_mask(vcpu, true); kvm_set_rflags(vcpu, X86_EFLAGS_FIXED); kvm_rip_write(vcpu, 0x8000); cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG); - kvm_x86_ops->set_cr0(vcpu, cr0); + kvm_x86_ops.set_cr0(vcpu, cr0); vcpu->arch.cr0 = cr0; - kvm_x86_ops->set_cr4(vcpu, 0); + kvm_x86_ops.set_cr4(vcpu, 0); /* Undocumented: IDT limit is set to zero on entry to SMM. */ dt.address = dt.size = 0; - kvm_x86_ops->set_idt(vcpu, &dt); + kvm_x86_ops.set_idt(vcpu, &dt); __kvm_set_dr(vcpu, 7, DR7_FIXED_1); @@ -7992,7 +8005,7 @@ static void enter_smm(struct kvm_vcpu *vcpu) #ifdef CONFIG_X86_64 if (guest_cpuid_has(vcpu, X86_FEATURE_LM)) - kvm_x86_ops->set_efer(vcpu, 0); + kvm_x86_ops.set_efer(vcpu, 0); #endif kvm_update_cpuid(vcpu); @@ -8030,7 +8043,7 @@ void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm); kvm_apic_update_apicv(vcpu); - kvm_x86_ops->refresh_apicv_exec_ctrl(vcpu); + kvm_x86_ops.refresh_apicv_exec_ctrl(vcpu); } EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv); @@ -8043,23 +8056,30 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv); */ void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) { - if (!kvm_x86_ops->check_apicv_inhibit_reasons || - !kvm_x86_ops->check_apicv_inhibit_reasons(bit)) + unsigned long old, new, expected; + + if (!kvm_x86_ops.check_apicv_inhibit_reasons || + !kvm_x86_ops.check_apicv_inhibit_reasons(bit)) return; - if (activate) { - if (!test_and_clear_bit(bit, &kvm->arch.apicv_inhibit_reasons) || - !kvm_apicv_activated(kvm)) - return; - } else { - if (test_and_set_bit(bit, &kvm->arch.apicv_inhibit_reasons) || - kvm_apicv_activated(kvm)) - return; - } + old = READ_ONCE(kvm->arch.apicv_inhibit_reasons); + do { + expected = new = old; + if (activate) + __clear_bit(bit, &new); + else + __set_bit(bit, &new); + if (new == old) + break; + old = cmpxchg(&kvm->arch.apicv_inhibit_reasons, expected, new); + } while (old != expected); + + if (!!old == !!new) + return; trace_kvm_apicv_update_request(activate, bit); - if (kvm_x86_ops->pre_update_apicv_exec_ctrl) - kvm_x86_ops->pre_update_apicv_exec_ctrl(kvm, activate); + if (kvm_x86_ops.pre_update_apicv_exec_ctrl) + kvm_x86_ops.pre_update_apicv_exec_ctrl(kvm, activate); kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE); } EXPORT_SYMBOL_GPL(kvm_request_apicv_update); @@ -8075,7 +8095,7 @@ static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors); else { if (vcpu->arch.apicv_active) - kvm_x86_ops->sync_pir_to_irr(vcpu); + kvm_x86_ops.sync_pir_to_irr(vcpu); if (ioapic_in_kernel(vcpu->kvm)) kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors); } @@ -8095,7 +8115,7 @@ static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors, vcpu_to_synic(vcpu)->vec_bitmap, 256); - kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap); + kvm_x86_ops.load_eoi_exitmap(vcpu, eoi_exit_bitmap); } int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, @@ -8122,13 +8142,13 @@ void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) if (!lapic_in_kernel(vcpu)) return; - if (!kvm_x86_ops->set_apic_access_page_addr) + if (!kvm_x86_ops.set_apic_access_page_addr) return; page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); if (is_error_page(page)) return; - kvm_x86_ops->set_apic_access_page_addr(vcpu, page_to_phys(page)); + kvm_x86_ops.set_apic_access_page_addr(vcpu, page_to_phys(page)); /* * Do not pin apic access page in memory, the MMU notifier @@ -8160,7 +8180,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_request_pending(vcpu)) { if (kvm_check_request(KVM_REQ_GET_VMCS12_PAGES, vcpu)) { - if (unlikely(!kvm_x86_ops->get_vmcs12_pages(vcpu))) { + if (unlikely(!kvm_x86_ops.get_vmcs12_pages(vcpu))) { r = 0; goto out; } @@ -8180,8 +8200,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) } if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu)) kvm_mmu_sync_roots(vcpu); - if (kvm_check_request(KVM_REQ_LOAD_CR3, vcpu)) - kvm_mmu_load_cr3(vcpu); + if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu)) + kvm_mmu_load_pgd(vcpu); if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) kvm_vcpu_flush_tlb(vcpu, true); if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { @@ -8266,7 +8286,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto out; } - if (inject_pending_event(vcpu, req_int_win) != 0) + if (inject_pending_event(vcpu) != 0) req_immediate_exit = true; else { /* Enable SMI/NMI/IRQ window open exits if needed. @@ -8284,12 +8304,12 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) * SMI. */ if (vcpu->arch.smi_pending && !is_smm(vcpu)) - if (!kvm_x86_ops->enable_smi_window(vcpu)) + if (!kvm_x86_ops.enable_smi_window(vcpu)) req_immediate_exit = true; if (vcpu->arch.nmi_pending) - kvm_x86_ops->enable_nmi_window(vcpu); + kvm_x86_ops.enable_nmi_window(vcpu); if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win) - kvm_x86_ops->enable_irq_window(vcpu); + kvm_x86_ops.enable_irq_window(vcpu); WARN_ON(vcpu->arch.exception.pending); } @@ -8306,7 +8326,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) preempt_disable(); - kvm_x86_ops->prepare_guest_switch(vcpu); + kvm_x86_ops.prepare_guest_switch(vcpu); /* * Disable IRQs before setting IN_GUEST_MODE. Posted interrupt @@ -8337,7 +8357,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) * notified with kvm_vcpu_kick. */ if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active) - kvm_x86_ops->sync_pir_to_irr(vcpu); + kvm_x86_ops.sync_pir_to_irr(vcpu); if (vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) || need_resched() || signal_pending(current)) { @@ -8352,7 +8372,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (req_immediate_exit) { kvm_make_request(KVM_REQ_EVENT, vcpu); - kvm_x86_ops->request_immediate_exit(vcpu); + kvm_x86_ops.request_immediate_exit(vcpu); } trace_kvm_entry(vcpu->vcpu_id); @@ -8372,7 +8392,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD; } - kvm_x86_ops->run(vcpu); + kvm_x86_ops.run(vcpu); /* * Do this here before restoring debug registers on the host. And @@ -8382,7 +8402,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) */ if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) { WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP); - kvm_x86_ops->sync_dirty_debug_regs(vcpu); + kvm_x86_ops.sync_dirty_debug_regs(vcpu); kvm_update_dr0123(vcpu); kvm_update_dr6(vcpu); kvm_update_dr7(vcpu); @@ -8404,7 +8424,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu->mode = OUTSIDE_GUEST_MODE; smp_wmb(); - kvm_x86_ops->handle_exit_irqoff(vcpu, &exit_fastpath); + kvm_x86_ops.handle_exit_irqoff(vcpu, &exit_fastpath); /* * Consume any pending interrupts, including the possible source of @@ -8447,12 +8467,11 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (vcpu->arch.apic_attention) kvm_lapic_sync_from_vapic(vcpu); - vcpu->arch.gpa_available = false; - r = kvm_x86_ops->handle_exit(vcpu, exit_fastpath); + r = kvm_x86_ops.handle_exit(vcpu, exit_fastpath); return r; cancel_injection: - kvm_x86_ops->cancel_injection(vcpu); + kvm_x86_ops.cancel_injection(vcpu); if (unlikely(vcpu->arch.apic_attention)) kvm_lapic_sync_from_vapic(vcpu); out: @@ -8462,13 +8481,13 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu) { if (!kvm_arch_vcpu_runnable(vcpu) && - (!kvm_x86_ops->pre_block || kvm_x86_ops->pre_block(vcpu) == 0)) { + (!kvm_x86_ops.pre_block || kvm_x86_ops.pre_block(vcpu) == 0)) { srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); kvm_vcpu_block(vcpu); vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); - if (kvm_x86_ops->post_block) - kvm_x86_ops->post_block(vcpu); + if (kvm_x86_ops.post_block) + kvm_x86_ops.post_block(vcpu); if (!kvm_check_request(KVM_REQ_UNHALT, vcpu)) return 1; @@ -8488,15 +8507,14 @@ static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu) break; default: return -EINTR; - break; } return 1; } static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu) { - if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) - kvm_x86_ops->check_nested_events(vcpu, false); + if (is_guest_mode(vcpu) && kvm_x86_ops.check_nested_events) + kvm_x86_ops.check_nested_events(vcpu); return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && !vcpu->arch.apf.halted); @@ -8652,7 +8670,7 @@ static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) kvm_save_current_fpu(vcpu->arch.user_fpu); - /* PKRU is separately restored in kvm_x86_ops->run. */ + /* PKRU is separately restored in kvm_x86_ops.run. */ __copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state, ~XFEATURE_MASK_PKRU); @@ -8757,7 +8775,7 @@ static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) * that usually, but some bad designed PV devices (vmware * backdoor interface) need this to work */ - emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt); + emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); vcpu->arch.emulate_regs_need_sync_to_vcpu = false; } regs->rax = kvm_rax_read(vcpu); @@ -8855,10 +8873,10 @@ static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - kvm_x86_ops->get_idt(vcpu, &dt); + kvm_x86_ops.get_idt(vcpu, &dt); sregs->idt.limit = dt.size; sregs->idt.base = dt.address; - kvm_x86_ops->get_gdt(vcpu, &dt); + kvm_x86_ops.get_gdt(vcpu, &dt); sregs->gdt.limit = dt.size; sregs->gdt.base = dt.address; @@ -8943,7 +8961,7 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, int reason, bool has_error_code, u32 error_code) { - struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; + struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; int ret; init_emulate_ctxt(vcpu); @@ -9005,10 +9023,10 @@ static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) dt.size = sregs->idt.limit; dt.address = sregs->idt.base; - kvm_x86_ops->set_idt(vcpu, &dt); + kvm_x86_ops.set_idt(vcpu, &dt); dt.size = sregs->gdt.limit; dt.address = sregs->gdt.base; - kvm_x86_ops->set_gdt(vcpu, &dt); + kvm_x86_ops.set_gdt(vcpu, &dt); vcpu->arch.cr2 = sregs->cr2; mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; @@ -9018,16 +9036,16 @@ static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) kvm_set_cr8(vcpu, sregs->cr8); mmu_reset_needed |= vcpu->arch.efer != sregs->efer; - kvm_x86_ops->set_efer(vcpu, sregs->efer); + kvm_x86_ops.set_efer(vcpu, sregs->efer); mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; - kvm_x86_ops->set_cr0(vcpu, sregs->cr0); + kvm_x86_ops.set_cr0(vcpu, sregs->cr0); vcpu->arch.cr0 = sregs->cr0; mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)); - kvm_x86_ops->set_cr4(vcpu, sregs->cr4); + kvm_x86_ops.set_cr4(vcpu, sregs->cr4); if (cpuid_update_needed) kvm_update_cpuid(vcpu); @@ -9133,7 +9151,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, */ kvm_set_rflags(vcpu, rflags); - kvm_x86_ops->update_bp_intercept(vcpu); + kvm_x86_ops.update_bp_intercept(vcpu); r = 0; @@ -9275,7 +9293,6 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) struct page *page; int r; - vcpu->arch.emulate_ctxt.ops = &emulate_ops; if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; else @@ -9313,11 +9330,14 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) GFP_KERNEL_ACCOUNT)) goto fail_free_mce_banks; + if (!alloc_emulate_ctxt(vcpu)) + goto free_wbinvd_dirty_mask; + vcpu->arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache, GFP_KERNEL_ACCOUNT); if (!vcpu->arch.user_fpu) { pr_err("kvm: failed to allocate userspace's fpu\n"); - goto free_wbinvd_dirty_mask; + goto free_emulate_ctxt; } vcpu->arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache, @@ -9342,7 +9362,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) kvm_hv_vcpu_init(vcpu); - r = kvm_x86_ops->vcpu_create(vcpu); + r = kvm_x86_ops.vcpu_create(vcpu); if (r) goto free_guest_fpu; @@ -9359,6 +9379,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu); free_user_fpu: kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu); +free_emulate_ctxt: + kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); free_wbinvd_dirty_mask: free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); fail_free_mce_banks: @@ -9393,11 +9415,9 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) mutex_unlock(&vcpu->mutex); - if (!kvmclock_periodic_sync) - return; - - schedule_delayed_work(&kvm->arch.kvmclock_sync_work, - KVMCLOCK_SYNC_PERIOD); + if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0) + schedule_delayed_work(&kvm->arch.kvmclock_sync_work, + KVMCLOCK_SYNC_PERIOD); } void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) @@ -9409,8 +9429,9 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) kvmclock_reset(vcpu); - kvm_x86_ops->vcpu_free(vcpu); + kvm_x86_ops.vcpu_free(vcpu); + kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu); kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu); @@ -9496,7 +9517,7 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) vcpu->arch.ia32_xss = 0; - kvm_x86_ops->vcpu_reset(vcpu, init_event); + kvm_x86_ops.vcpu_reset(vcpu, init_event); } void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) @@ -9521,7 +9542,7 @@ int kvm_arch_hardware_enable(void) bool stable, backwards_tsc = false; kvm_shared_msr_cpu_online(); - ret = kvm_x86_ops->hardware_enable(); + ret = kvm_x86_ops.hardware_enable(); if (ret != 0) return ret; @@ -9603,18 +9624,29 @@ int kvm_arch_hardware_enable(void) void kvm_arch_hardware_disable(void) { - kvm_x86_ops->hardware_disable(); + kvm_x86_ops.hardware_disable(); drop_user_return_notifiers(); } -int kvm_arch_hardware_setup(void) +int kvm_arch_hardware_setup(void *opaque) { + struct kvm_x86_init_ops *ops = opaque; int r; - r = kvm_x86_ops->hardware_setup(); + rdmsrl_safe(MSR_EFER, &host_efer); + + if (boot_cpu_has(X86_FEATURE_XSAVES)) + rdmsrl(MSR_IA32_XSS, host_xss); + + r = ops->hardware_setup(); if (r != 0) return r; + memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); + + if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) + supported_xss = 0; + cr4_reserved_bits = kvm_host_cr4_reserved_bits(&boot_cpu_data); if (kvm_has_tsc_control) { @@ -9631,28 +9663,26 @@ int kvm_arch_hardware_setup(void) kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits; } - if (boot_cpu_has(X86_FEATURE_XSAVES)) - rdmsrl(MSR_IA32_XSS, host_xss); - kvm_init_msr_list(); return 0; } void kvm_arch_hardware_unsetup(void) { - kvm_x86_ops->hardware_unsetup(); + kvm_x86_ops.hardware_unsetup(); } -int kvm_arch_check_processor_compat(void) +int kvm_arch_check_processor_compat(void *opaque) { struct cpuinfo_x86 *c = &cpu_data(smp_processor_id()); + struct kvm_x86_init_ops *ops = opaque; WARN_ON(!irqs_disabled()); if (kvm_host_cr4_reserved_bits(c) != cr4_reserved_bits) return -EIO; - return kvm_x86_ops->check_processor_compatibility(); + return ops->check_processor_compatibility(); } bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu) @@ -9678,9 +9708,16 @@ void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) pmu->need_cleanup = true; kvm_make_request(KVM_REQ_PMU, vcpu); } - kvm_x86_ops->sched_in(vcpu, cpu); + kvm_x86_ops.sched_in(vcpu, cpu); } +void kvm_arch_free_vm(struct kvm *kvm) +{ + kfree(kvm->arch.hyperv.hv_pa_pg); + vfree(kvm); +} + + int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { if (type) @@ -9715,7 +9752,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm_page_track_init(kvm); kvm_mmu_init_vm(kvm); - return kvm_x86_ops->vm_init(kvm); + return kvm_x86_ops.vm_init(kvm); } int kvm_arch_post_init_vm(struct kvm *kvm) @@ -9763,9 +9800,9 @@ void kvm_arch_sync_events(struct kvm *kvm) int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size) { int i, r; - unsigned long hva; + unsigned long hva, uninitialized_var(old_npages); struct kvm_memslots *slots = kvm_memslots(kvm); - struct kvm_memory_slot *slot, old; + struct kvm_memory_slot *slot; /* Called with kvm->slots_lock held. */ if (WARN_ON(id >= KVM_MEM_SLOTS_NUM)) @@ -9773,7 +9810,7 @@ int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size) slot = id_to_memslot(slots, id); if (size) { - if (slot->npages) + if (slot && slot->npages) return -EEXIST; /* @@ -9785,13 +9822,18 @@ int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size) if (IS_ERR((void *)hva)) return PTR_ERR((void *)hva); } else { - if (!slot->npages) + if (!slot || !slot->npages) return 0; - hva = 0; + /* + * Stuff a non-canonical value to catch use-after-delete. This + * ends up being 0 on 32-bit KVM, but there's no better + * alternative. + */ + hva = (unsigned long)(0xdeadull << 48); + old_npages = slot->npages; } - old = *slot; for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { struct kvm_userspace_memory_region m; @@ -9806,7 +9848,7 @@ int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size) } if (!size) - vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE); + vm_munmap(hva, old_npages * PAGE_SIZE); return 0; } @@ -9833,8 +9875,8 @@ void kvm_arch_destroy_vm(struct kvm *kvm) __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0); mutex_unlock(&kvm->slots_lock); } - if (kvm_x86_ops->vm_destroy) - kvm_x86_ops->vm_destroy(kvm); + if (kvm_x86_ops.vm_destroy) + kvm_x86_ops.vm_destroy(kvm); kvm_pic_destroy(kvm); kvm_ioapic_destroy(kvm); kvm_free_vcpus(kvm); @@ -9845,34 +9887,36 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_hv_destroy_vm(kvm); } -void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) +void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { int i; for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { - if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) { - kvfree(free->arch.rmap[i]); - free->arch.rmap[i] = NULL; - } + kvfree(slot->arch.rmap[i]); + slot->arch.rmap[i] = NULL; + if (i == 0) continue; - if (!dont || free->arch.lpage_info[i - 1] != - dont->arch.lpage_info[i - 1]) { - kvfree(free->arch.lpage_info[i - 1]); - free->arch.lpage_info[i - 1] = NULL; - } + kvfree(slot->arch.lpage_info[i - 1]); + slot->arch.lpage_info[i - 1] = NULL; } - kvm_page_track_free_memslot(free, dont); + kvm_page_track_free_memslot(slot); } -int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned long npages) +static int kvm_alloc_memslot_metadata(struct kvm_memory_slot *slot, + unsigned long npages) { int i; + /* + * Clear out the previous array pointers for the KVM_MR_MOVE case. The + * old arrays will be freed by __kvm_set_memory_region() if installing + * the new memslot is successful. + */ + memset(&slot->arch, 0, sizeof(slot->arch)); + for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { struct kvm_lpage_info *linfo; unsigned long ugfn; @@ -9903,11 +9947,9 @@ int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, ugfn = slot->userspace_addr >> PAGE_SHIFT; /* * If the gfn and userspace address are not aligned wrt each - * other, or if explicitly asked to, disable large page - * support for this slot + * other, disable large page support for this slot. */ - if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) || - !kvm_largepages_enabled()) { + if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) { unsigned long j; for (j = 0; j < lpages; ++j) @@ -9954,6 +9996,9 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, enum kvm_mr_change change) { + if (change == KVM_MR_CREATE || change == KVM_MR_MOVE) + return kvm_alloc_memslot_metadata(memslot, + mem->memory_size >> PAGE_SHIFT); return 0; } @@ -9962,14 +10007,14 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, { /* Still write protect RO slot */ if (new->flags & KVM_MEM_READONLY) { - kvm_mmu_slot_remove_write_access(kvm, new); + kvm_mmu_slot_remove_write_access(kvm, new, PT_PAGE_TABLE_LEVEL); return; } /* * Call kvm_x86_ops dirty logging hooks when they are valid. * - * kvm_x86_ops->slot_disable_log_dirty is called when: + * kvm_x86_ops.slot_disable_log_dirty is called when: * * - KVM_MR_CREATE with dirty logging is disabled * - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag @@ -9981,7 +10026,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, * any additional overhead from PML when guest is running with dirty * logging disabled for memory slots. * - * kvm_x86_ops->slot_enable_log_dirty is called when switching new slot + * kvm_x86_ops.slot_enable_log_dirty is called when switching new slot * to dirty logging mode. * * If kvm_x86_ops dirty logging hooks are invalid, use write protect. @@ -9997,19 +10042,32 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, * See the comments in fast_page_fault(). */ if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) { - if (kvm_x86_ops->slot_enable_log_dirty) - kvm_x86_ops->slot_enable_log_dirty(kvm, new); - else - kvm_mmu_slot_remove_write_access(kvm, new); + if (kvm_x86_ops.slot_enable_log_dirty) { + kvm_x86_ops.slot_enable_log_dirty(kvm, new); + } else { + int level = + kvm_dirty_log_manual_protect_and_init_set(kvm) ? + PT_DIRECTORY_LEVEL : PT_PAGE_TABLE_LEVEL; + + /* + * If we're with initial-all-set, we don't need + * to write protect any small page because + * they're reported as dirty already. However + * we still need to write-protect huge pages + * so that the page split can happen lazily on + * the first write to the huge page. + */ + kvm_mmu_slot_remove_write_access(kvm, new, level); + } } else { - if (kvm_x86_ops->slot_disable_log_dirty) - kvm_x86_ops->slot_disable_log_dirty(kvm, new); + if (kvm_x86_ops.slot_disable_log_dirty) + kvm_x86_ops.slot_disable_log_dirty(kvm, new); } } void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { @@ -10051,6 +10109,10 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, */ if (change != KVM_MR_DELETE) kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new); + + /* Free the arrays associated with the old memslot. */ + if (change == KVM_MR_MOVE) + kvm_arch_free_memslot(kvm, old); } void kvm_arch_flush_shadow_all(struct kvm *kvm) @@ -10067,8 +10129,8 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu) { return (is_guest_mode(vcpu) && - kvm_x86_ops->guest_apic_has_interrupt && - kvm_x86_ops->guest_apic_has_interrupt(vcpu)); + kvm_x86_ops.guest_apic_has_interrupt && + kvm_x86_ops.guest_apic_has_interrupt(vcpu)); } static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) @@ -10087,7 +10149,7 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) if (kvm_test_request(KVM_REQ_NMI, vcpu) || (vcpu->arch.nmi_pending && - kvm_x86_ops->nmi_allowed(vcpu))) + kvm_x86_ops.nmi_allowed(vcpu))) return true; if (kvm_test_request(KVM_REQ_SMI, vcpu) || @@ -10120,7 +10182,7 @@ bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) kvm_test_request(KVM_REQ_EVENT, vcpu)) return true; - if (vcpu->arch.apicv_active && kvm_x86_ops->dy_apicv_has_pending_interrupt(vcpu)) + if (vcpu->arch.apicv_active && kvm_x86_ops.dy_apicv_has_pending_interrupt(vcpu)) return true; return false; @@ -10138,7 +10200,7 @@ int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu) { - return kvm_x86_ops->interrupt_allowed(vcpu); + return kvm_x86_ops.interrupt_allowed(vcpu); } unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) @@ -10160,7 +10222,7 @@ unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) { unsigned long rflags; - rflags = kvm_x86_ops->get_rflags(vcpu); + rflags = kvm_x86_ops.get_rflags(vcpu); if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) rflags &= ~X86_EFLAGS_TF; return rflags; @@ -10172,7 +10234,7 @@ static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) rflags |= X86_EFLAGS_TF; - kvm_x86_ops->set_rflags(vcpu, rflags); + kvm_x86_ops.set_rflags(vcpu, rflags); } void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) @@ -10195,7 +10257,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) return; if (!vcpu->arch.mmu->direct_map && - work->arch.cr3 != vcpu->arch.mmu->get_cr3(vcpu)) + work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu)) return; kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true); @@ -10283,7 +10345,7 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) || (vcpu->arch.apf.send_user_only && - kvm_x86_ops->get_cpl(vcpu) == 0)) + kvm_x86_ops.get_cpl(vcpu) == 0)) return false; return true; @@ -10303,7 +10365,7 @@ bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu) * If interrupts are off we cannot even use an artificial * halt state. */ - return kvm_x86_ops->interrupt_allowed(vcpu); + return kvm_x86_ops.interrupt_allowed(vcpu); } void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, @@ -10432,7 +10494,7 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, irqfd->producer = prod; - return kvm_x86_ops->update_pi_irte(irqfd->kvm, + return kvm_x86_ops.update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 1); } @@ -10452,7 +10514,7 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, * when the irq is masked/disabled or the consumer side (KVM * int this case doesn't want to receive the interrupts. */ - ret = kvm_x86_ops->update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0); + ret = kvm_x86_ops.update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0); if (ret) printk(KERN_INFO "irq bypass consumer (token %p) unregistration" " fails: %d\n", irqfd->consumer.token, ret); @@ -10461,7 +10523,7 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, bool set) { - return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set); + return kvm_x86_ops.update_pi_irte(kvm, host_irq, guest_irq, set); } bool kvm_vector_hashing_enabled(void) @@ -10518,4 +10580,5 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi); +EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_update_request); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 3624665acee4..b968acc0516f 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -5,6 +5,7 @@ #include #include #include "kvm_cache_regs.h" +#include "kvm_emulate.h" #define KVM_DEFAULT_PLE_GAP 128 #define KVM_VMX_DEFAULT_PLE_WINDOW 4096 @@ -96,7 +97,7 @@ static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu) if (!is_long_mode(vcpu)) return false; - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l); return cs_l; } @@ -149,11 +150,6 @@ static inline u8 vcpu_virt_addr_bits(struct kvm_vcpu *vcpu) return kvm_read_cr4_bits(vcpu, X86_CR4_LA57) ? 57 : 48; } -static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt) -{ - return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48; -} - static inline u64 get_canonical(u64 la, u8 vaddr_bits) { return ((int64_t)la << (64 - vaddr_bits)) >> (64 - vaddr_bits); @@ -164,12 +160,6 @@ static inline bool is_noncanonical_address(u64 la, struct kvm_vcpu *vcpu) return get_canonical(la, vcpu_virt_addr_bits(vcpu)) != la; } -static inline bool emul_is_noncanonical_address(u64 la, - struct x86_emulate_ctxt *ctxt) -{ - return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la; -} - static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, unsigned access) { @@ -247,7 +237,7 @@ static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) static inline bool kvm_vcpu_latch_init(struct kvm_vcpu *vcpu) { - return is_smm(vcpu) || kvm_x86_ops->apic_init_signal_blocked(vcpu); + return is_smm(vcpu) || kvm_x86_ops.apic_init_signal_blocked(vcpu); } void kvm_set_pending_timer(struct kvm_vcpu *vcpu); @@ -280,13 +270,15 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, int emulation_type, void *insn, int insn_len); enum exit_fastpath_completion handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu); -#define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ - | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ - | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ - | XFEATURE_MASK_PKRU) extern u64 host_xcr0; +extern u64 supported_xcr0; +extern u64 supported_xss; -extern u64 kvm_supported_xcr0(void); +static inline bool kvm_mpx_supported(void) +{ + return (supported_xcr0 & (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR)) + == (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); +} extern unsigned int min_timer_period_us; diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 629fdf13f846..859519f5b342 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1310,7 +1310,7 @@ void do_user_addr_fault(struct pt_regs *regs, struct task_struct *tsk; struct mm_struct *mm; vm_fault_t fault, major = 0; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; tsk = current; mm = tsk->mm; @@ -1464,27 +1464,23 @@ void do_user_addr_fault(struct pt_regs *regs, fault = handle_mm_fault(vma, address, flags); major |= fault & VM_FAULT_MAJOR; + /* Quick path to respond to signals */ + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + no_context(regs, hw_error_code, address, SIGBUS, + BUS_ADRERR); + return; + } + /* * If we need to retry the mmap_sem has already been released, * and if there is a fatal signal pending there is no guarantee * that we made any progress. Handle this case first. */ - if (unlikely(fault & VM_FAULT_RETRY)) { - /* Retry at most once */ - if (flags & FAULT_FLAG_ALLOW_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; - flags |= FAULT_FLAG_TRIED; - if (!fatal_signal_pending(tsk)) - goto retry; - } - - /* User mode? Just return to handle the fatal exception */ - if (flags & FAULT_FLAG_USER) - return; - - /* Not returning to user mode? Handle exceptions or die: */ - no_context(regs, hw_error_code, address, SIGBUS, BUS_ADRERR); - return; + if (unlikely((fault & VM_FAULT_RETRY) && + (flags & FAULT_FLAG_ALLOW_RETRY))) { + flags |= FAULT_FLAG_TRIED; + goto retry; } up_read(&mm->mmap_sem); diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild index 271917c24b7f..9718e9593564 100644 --- a/arch/xtensa/include/asm/Kbuild +++ b/arch/xtensa/include/asm/Kbuild @@ -1,36 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += syscall_table.h -generic-y += bug.h -generic-y += compat.h -generic-y += device.h -generic-y += div64.h -generic-y += dma-mapping.h -generic-y += emergency-restart.h -generic-y += exec.h generic-y += extable.h -generic-y += fb.h -generic-y += hardirq.h -generic-y += hw_irq.h -generic-y += irq_regs.h -generic-y += irq_work.h -generic-y += kdebug.h -generic-y += kmap_types.h -generic-y += kprobes.h generic-y += kvm_para.h -generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h -generic-y += mm-arch-hooks.h -generic-y += mmiowb.h generic-y += param.h -generic-y += percpu.h -generic-y += preempt.h generic-y += qrwlock.h generic-y += qspinlock.h -generic-y += sections.h -generic-y += topology.h -generic-y += trace_clock.h generic-y += user.h -generic-y += vga.h -generic-y += word-at-a-time.h -generic-y += xor.h diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c index bee30a77cd70..e7172bd53ced 100644 --- a/arch/xtensa/mm/fault.c +++ b/arch/xtensa/mm/fault.c @@ -43,7 +43,7 @@ void do_page_fault(struct pt_regs *regs) int is_write, is_exec; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; code = SEGV_MAPERR; @@ -110,7 +110,7 @@ void do_page_fault(struct pt_regs *regs) */ fault = handle_mm_fault(vma, address, flags); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + if (fault_signal_pending(fault, regs)) return; if (unlikely(fault & VM_FAULT_ERROR)) { @@ -128,7 +128,6 @@ void do_page_fault(struct pt_regs *regs) else current->min_flt++; if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* No need to up_read(&mm->mmap_sem) as we would diff --git a/drivers/base/node.c b/drivers/base/node.c index 874a8b428438..502ab5447c8d 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -778,7 +778,7 @@ static int register_mem_sect_under_node(struct memory_block *mem_blk, * memory block could have several absent sections from start. * skip pfn range from absent section */ - if (!pfn_present(pfn)) { + if (!pfn_in_present_section(pfn)) { pfn = round_down(pfn + PAGES_PER_SECTION, PAGES_PER_SECTION) - 1; continue; diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 5142da401db3..092483644315 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -616,8 +616,8 @@ config TXX9_DMAC integrated in chips such as the Toshiba TX4927/38/39. config TEGRA20_APB_DMA - bool "NVIDIA Tegra20 APB DMA support" - depends on ARCH_TEGRA + tristate "NVIDIA Tegra20 APB DMA support" + depends on ARCH_TEGRA || COMPILE_TEST select DMA_ENGINE help Support for the NVIDIA Tegra20 APB DMA controller driver. The @@ -658,6 +658,17 @@ config UNIPHIER_MDMAC UniPhier platform. This DMA controller is used as the external DMA engine of the SD/eMMC controllers of the LD4, Pro4, sLD8 SoCs. +config UNIPHIER_XDMAC + tristate "UniPhier XDMAC support" + depends on ARCH_UNIPHIER || COMPILE_TEST + depends on OF + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the XDMAC (external DMA controller) on the + UniPhier platform. This DMA controller can transfer data from + memory to memory, memory to peripheral and peripheral to memory. + config XGENE_DMA tristate "APM X-Gene DMA support" depends on ARCH_XGENE || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 1d908394fbea..e60f81331d4c 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -78,6 +78,7 @@ obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o obj-$(CONFIG_TEGRA210_ADMA) += tegra210-adma.o obj-$(CONFIG_TIMB_DMA) += timb_dma.o obj-$(CONFIG_UNIPHIER_MDMAC) += uniphier-mdmac.o +obj-$(CONFIG_UNIPHIER_XDMAC) += uniphier-xdmac.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o obj-$(CONFIG_ZX_DMA) += zx_dma.o obj-$(CONFIG_ST_FDMA) += st_fdma.o diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 672c73b4a2d4..73a20780744b 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -146,17 +146,8 @@ static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) "scanned %u descriptors on freelist\n", i); /* no more descriptor available in initial pool: create one more */ - if (!ret) { - ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); - if (ret) { - spin_lock_irqsave(&atchan->lock, flags); - atchan->descs_allocated++; - spin_unlock_irqrestore(&atchan->lock, flags); - } else { - dev_err(chan2dev(&atchan->chan_common), - "not enough descriptors available\n"); - } - } + if (!ret) + ret = atc_alloc_descriptor(&atchan->chan_common, GFP_NOWAIT); return ret; } @@ -435,17 +426,19 @@ static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie) * atc_chain_complete - finish work for one transaction chain * @atchan: channel we work on * @desc: descriptor at the head of the chain we want do complete - * - * Called with atchan->lock held and bh disabled */ + */ static void atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) { struct dma_async_tx_descriptor *txd = &desc->txd; struct at_dma *atdma = to_at_dma(atchan->chan_common.device); + unsigned long flags; dev_vdbg(chan2dev(&atchan->chan_common), "descriptor %u complete\n", txd->cookie); + spin_lock_irqsave(&atchan->lock, flags); + /* mark the descriptor as complete for non cyclic cases only */ if (!atc_chan_is_cyclic(atchan)) dma_cookie_complete(txd); @@ -462,16 +455,13 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) /* move myself to free_list */ list_move(&desc->desc_node, &atchan->free_list); + spin_unlock_irqrestore(&atchan->lock, flags); + dma_descriptor_unmap(txd); /* for cyclic transfers, * no need to replay callback function while stopping */ - if (!atc_chan_is_cyclic(atchan)) { - /* - * The API requires that no submissions are done from a - * callback, so we don't need to drop the lock here - */ + if (!atc_chan_is_cyclic(atchan)) dmaengine_desc_get_callback_invoke(txd, NULL); - } dma_run_dependencies(txd); } @@ -489,9 +479,12 @@ static void atc_complete_all(struct at_dma_chan *atchan) { struct at_desc *desc, *_desc; LIST_HEAD(list); + unsigned long flags; dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); + spin_lock_irqsave(&atchan->lock, flags); + /* * Submit queued descriptors ASAP, i.e. before we go through * the completed ones. @@ -503,6 +496,8 @@ static void atc_complete_all(struct at_dma_chan *atchan) /* empty queue list by moving descriptors (if any) to active_list */ list_splice_init(&atchan->queue, &atchan->active_list); + spin_unlock_irqrestore(&atchan->lock, flags); + list_for_each_entry_safe(desc, _desc, &list, desc_node) atc_chain_complete(atchan, desc); } @@ -510,38 +505,44 @@ static void atc_complete_all(struct at_dma_chan *atchan) /** * atc_advance_work - at the end of a transaction, move forward * @atchan: channel where the transaction ended - * - * Called with atchan->lock held and bh disabled */ static void atc_advance_work(struct at_dma_chan *atchan) { + unsigned long flags; + int ret; + dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); - if (atc_chan_is_enabled(atchan)) + spin_lock_irqsave(&atchan->lock, flags); + ret = atc_chan_is_enabled(atchan); + spin_unlock_irqrestore(&atchan->lock, flags); + if (ret) return; if (list_empty(&atchan->active_list) || - list_is_singular(&atchan->active_list)) { - atc_complete_all(atchan); - } else { - atc_chain_complete(atchan, atc_first_active(atchan)); - /* advance work */ - atc_dostart(atchan, atc_first_active(atchan)); - } + list_is_singular(&atchan->active_list)) + return atc_complete_all(atchan); + + atc_chain_complete(atchan, atc_first_active(atchan)); + + /* advance work */ + spin_lock_irqsave(&atchan->lock, flags); + atc_dostart(atchan, atc_first_active(atchan)); + spin_unlock_irqrestore(&atchan->lock, flags); } /** * atc_handle_error - handle errors reported by DMA controller * @atchan: channel where error occurs - * - * Called with atchan->lock held and bh disabled */ static void atc_handle_error(struct at_dma_chan *atchan) { struct at_desc *bad_desc; struct at_desc *child; + unsigned long flags; + spin_lock_irqsave(&atchan->lock, flags); /* * The descriptor currently at the head of the active list is * broked. Since we don't have any way to report errors, we'll @@ -573,6 +574,8 @@ static void atc_handle_error(struct at_dma_chan *atchan) list_for_each_entry(child, &bad_desc->tx_list, desc_node) atc_dump_lli(atchan, &child->lli); + spin_unlock_irqrestore(&atchan->lock, flags); + /* Pretend the descriptor completed successfully */ atc_chain_complete(atchan, bad_desc); } @@ -580,8 +583,6 @@ static void atc_handle_error(struct at_dma_chan *atchan) /** * atc_handle_cyclic - at the end of a period, run callback function * @atchan: channel used for cyclic operations - * - * Called with atchan->lock held and bh disabled */ static void atc_handle_cyclic(struct at_dma_chan *atchan) { @@ -600,17 +601,14 @@ static void atc_handle_cyclic(struct at_dma_chan *atchan) static void atc_tasklet(unsigned long data) { struct at_dma_chan *atchan = (struct at_dma_chan *)data; - unsigned long flags; - spin_lock_irqsave(&atchan->lock, flags); if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) - atc_handle_error(atchan); - else if (atc_chan_is_cyclic(atchan)) - atc_handle_cyclic(atchan); - else - atc_advance_work(atchan); + return atc_handle_error(atchan); - spin_unlock_irqrestore(&atchan->lock, flags); + if (atc_chan_is_cyclic(atchan)) + return atc_handle_cyclic(atchan); + + atc_advance_work(atchan); } static irqreturn_t at_dma_interrupt(int irq, void *dev_id) @@ -940,7 +938,7 @@ atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, return NULL; } - vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr); + vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr); if (!vaddr) { dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", __func__); @@ -998,7 +996,7 @@ atc_prep_dma_memset_sg(struct dma_chan *chan, return NULL; } - vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr); + vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr); if (!vaddr) { dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", __func__); @@ -1446,6 +1444,8 @@ static int atc_terminate_all(struct dma_chan *chan) list_splice_init(&atchan->queue, &list); list_splice_init(&atchan->active_list, &list); + spin_unlock_irqrestore(&atchan->lock, flags); + /* Flush all pending and queued descriptors */ list_for_each_entry_safe(desc, _desc, &list, desc_node) atc_chain_complete(atchan, desc); @@ -1454,8 +1454,6 @@ static int atc_terminate_all(struct dma_chan *chan) /* if channel dedicated to cyclic operations, free it */ clear_bit(ATC_IS_CYCLIC, &atchan->status); - spin_unlock_irqrestore(&atchan->lock, flags); - return 0; } @@ -1516,7 +1514,6 @@ atc_tx_status(struct dma_chan *chan, static void atc_issue_pending(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); - unsigned long flags; dev_vdbg(chan2dev(chan), "issue_pending\n"); @@ -1524,15 +1521,12 @@ static void atc_issue_pending(struct dma_chan *chan) if (atc_chan_is_cyclic(atchan)) return; - spin_lock_irqsave(&atchan->lock, flags); atc_advance_work(atchan); - spin_unlock_irqrestore(&atchan->lock, flags); } /** * atc_alloc_chan_resources - allocate resources for DMA channel * @chan: allocate descriptor resources for this channel - * @client: current client requesting the channel be ready for requests * * return - the number of allocated descriptors */ @@ -1542,10 +1536,8 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) struct at_dma *atdma = to_at_dma(chan->device); struct at_desc *desc; struct at_dma_slave *atslave; - unsigned long flags; int i; u32 cfg; - LIST_HEAD(tmp_list); dev_vdbg(chan2dev(chan), "alloc_chan_resources\n"); @@ -1555,6 +1547,11 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) return -EIO; } + if (!list_empty(&atchan->free_list)) { + dev_dbg(chan2dev(chan), "can't allocate channel resources (channel not freed from a previous use)\n"); + return -EIO; + } + cfg = ATC_DEFAULT_CFG; atslave = chan->private; @@ -1570,11 +1567,6 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) cfg = atslave->cfg; } - /* have we already been set up? - * reconfigure channel but no need to reallocate descriptors */ - if (!list_empty(&atchan->free_list)) - return atchan->descs_allocated; - /* Allocate initial pool of descriptors */ for (i = 0; i < init_nr_desc_per_channel; i++) { desc = atc_alloc_descriptor(chan, GFP_KERNEL); @@ -1583,23 +1575,18 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) "Only %d initial descriptors\n", i); break; } - list_add_tail(&desc->desc_node, &tmp_list); + list_add_tail(&desc->desc_node, &atchan->free_list); } - spin_lock_irqsave(&atchan->lock, flags); - atchan->descs_allocated = i; - list_splice(&tmp_list, &atchan->free_list); dma_cookie_init(chan); - spin_unlock_irqrestore(&atchan->lock, flags); /* channel parameters */ channel_writel(atchan, CFG, cfg); dev_dbg(chan2dev(chan), - "alloc_chan_resources: allocated %d descriptors\n", - atchan->descs_allocated); + "alloc_chan_resources: allocated %d descriptors\n", i); - return atchan->descs_allocated; + return i; } /** @@ -1613,9 +1600,6 @@ static void atc_free_chan_resources(struct dma_chan *chan) struct at_desc *desc, *_desc; LIST_HEAD(list); - dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n", - atchan->descs_allocated); - /* ASSERT: channel is idle */ BUG_ON(!list_empty(&atchan->active_list)); BUG_ON(!list_empty(&atchan->queue)); @@ -1628,7 +1612,6 @@ static void atc_free_chan_resources(struct dma_chan *chan) dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys); } list_splice_init(&atchan->free_list, &list); - atchan->descs_allocated = 0; atchan->status = 0; /* @@ -1671,7 +1654,7 @@ static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec, dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); - atslave = kzalloc(sizeof(*atslave), GFP_KERNEL); + atslave = kmalloc(sizeof(*atslave), GFP_KERNEL); if (!atslave) return NULL; diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h index fe8a5853ec49..397692e937b3 100644 --- a/drivers/dma/at_hdmac_regs.h +++ b/drivers/dma/at_hdmac_regs.h @@ -243,7 +243,6 @@ enum atc_status { * @active_list: list of descriptors dmaengine is being running on * @queue: list of descriptors ready to be submitted to engine * @free_list: list of descriptors usable by the channel - * @descs_allocated: records the actual size of the descriptor pool */ struct at_dma_chan { struct dma_chan chan_common; @@ -264,7 +263,6 @@ struct at_dma_chan { struct list_head active_list; struct list_head queue; struct list_head free_list; - unsigned int descs_allocated; }; #define channel_readl(atchan, name) \ diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index f71c9f77d405..bb0eaf38b594 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -1543,9 +1543,6 @@ static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan, static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) { struct at_xdmac_desc *desc; - unsigned long flags; - - spin_lock_irqsave(&atchan->lock, flags); /* * If channel is enabled, do nothing, advance_work will be triggered @@ -1559,8 +1556,6 @@ static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) if (!desc->active_xfer) at_xdmac_start_xfer(atchan, desc); } - - spin_unlock_irqrestore(&atchan->lock, flags); } static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) @@ -1596,7 +1591,7 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) if (atchan->irq_status & AT_XDMAC_CIS_ROIS) dev_err(chan2dev(&atchan->chan), "request overflow error!!!"); - spin_lock_bh(&atchan->lock); + spin_lock_irq(&atchan->lock); /* Channel must be disabled first as it's not done automatically */ at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); @@ -1607,7 +1602,7 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) struct at_xdmac_desc, xfer_node); - spin_unlock_bh(&atchan->lock); + spin_unlock_irq(&atchan->lock); /* Print bad descriptor's details if needed */ dev_dbg(chan2dev(&atchan->chan), @@ -1640,31 +1635,31 @@ static void at_xdmac_tasklet(unsigned long data) if (atchan->irq_status & error_mask) at_xdmac_handle_error(atchan); - spin_lock(&atchan->lock); + spin_lock_irq(&atchan->lock); desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); if (!desc->active_xfer) { dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting"); - spin_unlock(&atchan->lock); + spin_unlock_irq(&atchan->lock); return; } txd = &desc->tx_dma_desc; at_xdmac_remove_xfer(atchan, desc); - spin_unlock(&atchan->lock); + spin_unlock_irq(&atchan->lock); - if (!at_xdmac_chan_is_cyclic(atchan)) { - dma_cookie_complete(txd); - if (txd->flags & DMA_PREP_INTERRUPT) - dmaengine_desc_get_callback_invoke(txd, NULL); - } + dma_cookie_complete(txd); + if (txd->flags & DMA_PREP_INTERRUPT) + dmaengine_desc_get_callback_invoke(txd, NULL); dma_run_dependencies(txd); + spin_lock_irq(&atchan->lock); at_xdmac_advance_work(atchan); + spin_unlock_irq(&atchan->lock); } } @@ -1725,11 +1720,15 @@ static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id) static void at_xdmac_issue_pending(struct dma_chan *chan) { struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + unsigned long flags; dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__); - if (!at_xdmac_chan_is_cyclic(atchan)) + if (!at_xdmac_chan_is_cyclic(atchan)) { + spin_lock_irqsave(&atchan->lock, flags); at_xdmac_advance_work(atchan); + spin_unlock_irqrestore(&atchan->lock, flags); + } return; } @@ -1822,26 +1821,21 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac_desc *desc; int i; - unsigned long flags; - - spin_lock_irqsave(&atchan->lock, flags); if (at_xdmac_chan_is_enabled(atchan)) { dev_err(chan2dev(chan), "can't allocate channel resources (channel enabled)\n"); - i = -EIO; - goto spin_unlock; + return -EIO; } if (!list_empty(&atchan->free_descs_list)) { dev_err(chan2dev(chan), "can't allocate channel resources (channel not free from a previous use)\n"); - i = -EIO; - goto spin_unlock; + return -EIO; } for (i = 0; i < init_nr_desc_per_channel; i++) { - desc = at_xdmac_alloc_desc(chan, GFP_ATOMIC); + desc = at_xdmac_alloc_desc(chan, GFP_KERNEL); if (!desc) { dev_warn(chan2dev(chan), "only %d descriptors have been allocated\n", i); @@ -1854,8 +1848,6 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i); -spin_unlock: - spin_unlock_irqrestore(&atchan->lock, flags); return i; } diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index 275e90fa829d..64239da02e74 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -120,7 +120,7 @@ struct sba_request { struct brcm_message msg; struct dma_async_tx_descriptor tx; /* SBA commands */ - struct brcm_sba_command cmds[0]; + struct brcm_sba_command cmds[]; }; enum sba_version { diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 17909fd1820f..4830ba658ce1 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -58,6 +58,87 @@ static DEFINE_IDA(dma_ida); static LIST_HEAD(dma_device_list); static long dmaengine_ref_count; +/* --- debugfs implementation --- */ +#ifdef CONFIG_DEBUG_FS +#include + +static struct dentry *rootdir; + +static void dmaengine_debug_register(struct dma_device *dma_dev) +{ + dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev), + rootdir); + if (IS_ERR(dma_dev->dbg_dev_root)) + dma_dev->dbg_dev_root = NULL; +} + +static void dmaengine_debug_unregister(struct dma_device *dma_dev) +{ + debugfs_remove_recursive(dma_dev->dbg_dev_root); + dma_dev->dbg_dev_root = NULL; +} + +static void dmaengine_dbg_summary_show(struct seq_file *s, + struct dma_device *dma_dev) +{ + struct dma_chan *chan; + + list_for_each_entry(chan, &dma_dev->channels, device_node) { + if (chan->client_count) { + seq_printf(s, " %-13s| %s", dma_chan_name(chan), + chan->dbg_client_name ?: "in-use"); + + if (chan->router) + seq_printf(s, " (via router: %s)\n", + dev_name(chan->router->dev)); + else + seq_puts(s, "\n"); + } + } +} + +static int dmaengine_summary_show(struct seq_file *s, void *data) +{ + struct dma_device *dma_dev = NULL; + + mutex_lock(&dma_list_mutex); + list_for_each_entry(dma_dev, &dma_device_list, global_node) { + seq_printf(s, "dma%d (%s): number of channels: %u\n", + dma_dev->dev_id, dev_name(dma_dev->dev), + dma_dev->chancnt); + + if (dma_dev->dbg_summary_show) + dma_dev->dbg_summary_show(s, dma_dev); + else + dmaengine_dbg_summary_show(s, dma_dev); + + if (!list_is_last(&dma_dev->global_node, &dma_device_list)) + seq_puts(s, "\n"); + } + mutex_unlock(&dma_list_mutex); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dmaengine_summary); + +static void __init dmaengine_debugfs_init(void) +{ + rootdir = debugfs_create_dir("dmaengine", NULL); + + /* /sys/kernel/debug/dmaengine/summary */ + debugfs_create_file("summary", 0444, rootdir, NULL, + &dmaengine_summary_fops); +} +#else +static inline void dmaengine_debugfs_init(void) { } +static inline int dmaengine_debug_register(struct dma_device *dma_dev) +{ + return 0; +} + +static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { } +#endif /* DEBUG_FS */ + /* --- sysfs implementation --- */ #define DMA_SLAVE_NAME "slave" @@ -760,6 +841,11 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name) return chan ? chan : ERR_PTR(-EPROBE_DEFER); found: +#ifdef CONFIG_DEBUG_FS + chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), + name); +#endif + chan->name = kasprintf(GFP_KERNEL, "dma:%s", name); if (!chan->name) return chan; @@ -837,6 +923,11 @@ void dma_release_channel(struct dma_chan *chan) chan->name = NULL; chan->slave = NULL; } + +#ifdef CONFIG_DEBUG_FS + kfree(chan->dbg_client_name); + chan->dbg_client_name = NULL; +#endif mutex_unlock(&dma_list_mutex); } EXPORT_SYMBOL_GPL(dma_release_channel); @@ -1196,6 +1287,8 @@ int dma_async_device_register(struct dma_device *device) dma_channel_rebalance(); mutex_unlock(&dma_list_mutex); + dmaengine_debug_register(device); + return 0; err_out: @@ -1229,6 +1322,8 @@ void dma_async_device_unregister(struct dma_device *device) { struct dma_chan *chan, *n; + dmaengine_debug_unregister(device); + list_for_each_entry_safe(chan, n, &device->channels, device_node) __dma_async_device_channel_unregister(device, chan); @@ -1559,6 +1654,11 @@ static int __init dma_bus_init(void) if (err) return err; - return class_register(&dma_devclass); + + err = class_register(&dma_devclass); + if (!err) + dmaengine_debugfs_init(); + + return err; } arch_initcall(dma_bus_init); diff --git a/drivers/dma/dmaengine.h b/drivers/dma/dmaengine.h index e8a320c9e57c..1bfbd64b1371 100644 --- a/drivers/dma/dmaengine.h +++ b/drivers/dma/dmaengine.h @@ -182,4 +182,20 @@ dmaengine_desc_callback_valid(struct dmaengine_desc_callback *cb) struct dma_chan *dma_get_slave_channel(struct dma_chan *chan); struct dma_chan *dma_get_any_slave_channel(struct dma_device *device); +#ifdef CONFIG_DEBUG_FS +#include + +static inline struct dentry * +dmaengine_get_debugfs_root(struct dma_device *dma_dev) { + return dma_dev->dbg_dev_root; +} +#else +struct dentry; +static inline struct dentry * +dmaengine_get_debugfs_root(struct dma_device *dma_dev) +{ + return NULL; +} +#endif /* CONFIG_DEBUG_FS */ + #endif diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c index c70a7965f140..4ec909e0b810 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c @@ -790,6 +790,20 @@ static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) return 0; } +static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev) +{ + struct dpaa2_qdma_priv *priv; + struct device *dev; + + dev = &ls_dev->dev; + priv = dev_get_drvdata(dev); + + dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); + dpaa2_dpdmai_dpio_unbind(priv); + dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); + dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle); +} + static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { { .vendor = FSL_MC_VENDOR_FREESCALE, @@ -805,6 +819,7 @@ static struct fsl_mc_driver dpaa2_qdma_driver = { }, .probe = dpaa2_qdma_probe, .remove = dpaa2_qdma_remove, + .shutdown = dpaa2_qdma_shutdown, .match_id_table = dpaa2_qdma_id_table }; diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c index f8d22115154a..878662aaa1c2 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c @@ -159,6 +159,27 @@ int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, return 0; } +/** + * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * Return: '0' on Success; error code otherwise. + */ +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} +EXPORT_SYMBOL_GPL(dpdmai_destroy); + /** * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames. * @mc_io: Pointer to MC portal's I/O object diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h index 6d785093da8e..b13b9bf0c003 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h @@ -18,6 +18,7 @@ #define DPDMAI_CMDID_CLOSE DPDMAI_CMDID_FORMAT(0x800) #define DPDMAI_CMDID_OPEN DPDMAI_CMDID_FORMAT(0x80E) #define DPDMAI_CMDID_CREATE DPDMAI_CMDID_FORMAT(0x90E) +#define DPDMAI_CMDID_DESTROY DPDMAI_CMDID_FORMAT(0x900) #define DPDMAI_CMDID_ENABLE DPDMAI_CMDID_FORMAT(0x002) #define DPDMAI_CMDID_DISABLE DPDMAI_CMDID_FORMAT(0x003) @@ -160,6 +161,7 @@ struct dpdmai_rx_queue_attr { int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpdmai_id, u16 *token); int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, const struct dpdmai_cfg *cfg, u16 *token); int dpdmai_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 989b7a25ca61..ff49847e37a8 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -74,12 +74,10 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) struct idxd_device *idxd; struct idxd_wq *wq; struct device *dev; - struct idxd_cdev *idxd_cdev; wq = inode_wq(inode); idxd = wq->idxd; dev = &idxd->pdev->dev; - idxd_cdev = &wq->idxd_cdev; dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); @@ -139,6 +137,8 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) dev_dbg(&pdev->dev, "%s called\n", __func__); rc = check_vma(wq, vma, __func__); + if (rc < 0) + return rc; vma->vm_flags |= VM_DONTCOPY; pfn = (base + idxd_get_wq_portal_full_offset(wq->id, diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index ada69e722f84..f6f49f0f6fae 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -584,11 +584,11 @@ static void idxd_group_flags_setup(struct idxd_device *idxd) struct idxd_group *group = &idxd->groups[i]; if (group->tc_a == -1) - group->grpcfg.flags.tc_a = 0; + group->tc_a = group->grpcfg.flags.tc_a = 0; else group->grpcfg.flags.tc_a = group->tc_a; if (group->tc_b == -1) - group->grpcfg.flags.tc_b = 1; + group->tc_b = group->grpcfg.flags.tc_b = 1; else group->grpcfg.flags.tc_b = group->tc_b; group->grpcfg.flags.use_token_limit = group->use_token_limit; diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 6ca6e520a2fa..3999827970ab 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -419,7 +419,7 @@ static ssize_t engine_group_id_store(struct device *dev, struct idxd_device *idxd = engine->idxd; long id; int rc; - struct idxd_group *prevg, *group; + struct idxd_group *prevg; rc = kstrtol(buf, 10, &id); if (rc < 0) @@ -439,7 +439,6 @@ static ssize_t engine_group_id_store(struct device *dev, return count; } - group = &idxd->groups[id]; prevg = engine->group; if (prevg) @@ -513,9 +512,6 @@ static ssize_t group_tokens_reserved_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; - if (idxd->token_limit == 0) - return -EPERM; - if (val > idxd->max_tokens) return -EINVAL; @@ -561,8 +557,6 @@ static ssize_t group_tokens_allowed_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; - if (idxd->token_limit == 0) - return -EPERM; if (val < 4 * group->num_engines || val > group->tokens_reserved + idxd->nr_tokens) return -EINVAL; @@ -1180,6 +1174,16 @@ static ssize_t op_cap_show(struct device *dev, } static DEVICE_ATTR_RO(op_cap); +static ssize_t gen_cap_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_device *idxd = + container_of(dev, struct idxd_device, conf_dev); + + return sprintf(buf, "%#llx\n", idxd->hw.gen_cap.bits); +} +static DEVICE_ATTR_RO(gen_cap); + static ssize_t configurable_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1317,6 +1321,7 @@ static struct attribute *idxd_device_attributes[] = { &dev_attr_max_batch_size.attr, &dev_attr_max_transfer_size.attr, &dev_attr_op_cap.attr, + &dev_attr_gen_cap.attr, &dev_attr_configurable.attr, &dev_attr_clients.attr, &dev_attr_state.attr, diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index be61c32a876f..0be385587c4c 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -102,7 +102,7 @@ struct ioat_dca_priv { int max_requesters; int requester_count; u8 tag_map[IOAT_TAG_MAP_LEN]; - struct ioat_dca_slot req_slots[0]; + struct ioat_dca_slot req_slots[]; }; static int ioat_dca_dev_managed(struct dca_provider *dca, diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index fbabd2e88a18..4db000d5f01c 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -4303,7 +4303,7 @@ static ssize_t devices_show(struct device_driver *dev, char *buf) for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { if (ppc440spe_adma_devices[i] == -1) continue; - size += snprintf(buf + size, PAGE_SIZE - size, + size += scnprintf(buf + size, PAGE_SIZE - size, "PPC440SP(E)-ADMA.%d: %s\n", i, ppc_adma_errors[ppc440spe_adma_devices[i]]); } diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c index afb68055ed1b..0fa7f14a65a1 100644 --- a/drivers/dma/sa11x0-dma.c +++ b/drivers/dma/sa11x0-dma.c @@ -78,7 +78,7 @@ struct sa11x0_dma_desc { bool cyclic; unsigned sglen; - struct sa11x0_dma_sg sg[0]; + struct sa11x0_dma_sg sg[]; }; struct sa11x0_dma_phy; diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index f06016d38a05..59b36ab5d684 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -1219,7 +1219,7 @@ rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, sg_len = buf_len / period_len; if (sg_len > RCAR_DMAC_MAX_SG_LEN) { dev_err(chan->device->dev, - "chan%u: sg length %d exceds limit %d", + "chan%u: sg length %d exceeds limit %d", rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN); return NULL; } diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c index c51de498b5b4..2deeaab078a4 100644 --- a/drivers/dma/sh/shdma-base.c +++ b/drivers/dma/sh/shdma-base.c @@ -709,7 +709,7 @@ static struct dma_async_tx_descriptor *shdma_prep_dma_cyclic( BUG_ON(!schan->desc_num); if (sg_len > SHDMA_MAX_SG_LEN) { - dev_err(schan->dev, "sg length %d exceds limit %d", + dev_err(schan->dev, "sg length %d exceeds limit %d", sg_len, SHDMA_MAX_SG_LEN); return NULL; } diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c index 9a31a315dbef..0ef5ca81ba4d 100644 --- a/drivers/dma/sprd-dma.c +++ b/drivers/dma/sprd-dma.c @@ -212,7 +212,7 @@ struct sprd_dma_dev { struct clk *ashb_clk; int irq; u32 total_chns; - struct sprd_dma_chn channels[0]; + struct sprd_dma_chn channels[]; }; static void sprd_dma_free_desc(struct virt_dma_desc *vd); @@ -486,6 +486,28 @@ static int sprd_dma_set_2stage_config(struct sprd_dma_chn *schan) return 0; } +static void sprd_dma_set_pending(struct sprd_dma_chn *schan, bool enable) +{ + struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); + u32 reg, val, req_id; + + if (schan->dev_id == SPRD_DMA_SOFTWARE_UID) + return; + + /* The DMA request id always starts from 0. */ + req_id = schan->dev_id - 1; + + if (req_id < 32) { + reg = SPRD_DMA_GLB_REQ_PEND0_EN; + val = BIT(req_id); + } else { + reg = SPRD_DMA_GLB_REQ_PEND1_EN; + val = BIT(req_id - 32); + } + + sprd_dma_glb_update(sdev, reg, val, enable ? val : 0); +} + static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan, struct sprd_dma_desc *sdesc) { @@ -532,6 +554,7 @@ static void sprd_dma_start(struct sprd_dma_chn *schan) */ sprd_dma_set_chn_config(schan, schan->cur_desc); sprd_dma_set_uid(schan); + sprd_dma_set_pending(schan, true); sprd_dma_enable_chn(schan); if (schan->dev_id == SPRD_DMA_SOFTWARE_UID && @@ -543,6 +566,7 @@ static void sprd_dma_start(struct sprd_dma_chn *schan) static void sprd_dma_stop(struct sprd_dma_chn *schan) { sprd_dma_stop_and_disable(schan); + sprd_dma_set_pending(schan, false); sprd_dma_unset_uid(schan); sprd_dma_clear_int(schan); schan->cur_desc = NULL; diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index 5989b0893521..0ddbaa4b4f0b 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -207,7 +208,6 @@ struct stm32_dma_device { struct dma_device ddev; void __iomem *base; struct clk *clk; - struct reset_control *rst; bool mem2mem; struct stm32_dma_chan chan[STM32_DMA_MAX_CHANNELS]; }; @@ -422,29 +422,19 @@ static void stm32_dma_irq_clear(struct stm32_dma_chan *chan, u32 flags) static int stm32_dma_disable_chan(struct stm32_dma_chan *chan) { struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); - unsigned long timeout = jiffies + msecs_to_jiffies(5000); - u32 dma_scr, id; + u32 dma_scr, id, reg; id = chan->id; - dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); + reg = STM32_DMA_SCR(id); + dma_scr = stm32_dma_read(dmadev, reg); if (dma_scr & STM32_DMA_SCR_EN) { dma_scr &= ~STM32_DMA_SCR_EN; - stm32_dma_write(dmadev, STM32_DMA_SCR(id), dma_scr); + stm32_dma_write(dmadev, reg, dma_scr); - do { - dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); - dma_scr &= STM32_DMA_SCR_EN; - if (!dma_scr) - break; - - if (time_after_eq(jiffies, timeout)) { - dev_err(chan2dev(chan), "%s: timeout!\n", - __func__); - return -EBUSY; - } - cond_resched(); - } while (1); + return readl_relaxed_poll_timeout_atomic(dmadev->base + reg, + dma_scr, !(dma_scr & STM32_DMA_SCR_EN), + 10, 1000000); } return 0; @@ -488,8 +478,10 @@ static int stm32_dma_terminate_all(struct dma_chan *c) spin_lock_irqsave(&chan->vchan.lock, flags); - if (chan->busy) { - stm32_dma_stop(chan); + if (chan->desc) { + vchan_terminate_vdesc(&chan->desc->vdesc); + if (chan->busy) + stm32_dma_stop(chan); chan->desc = NULL; } @@ -545,6 +537,8 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) if (!vdesc) return; + list_del(&vdesc->node); + chan->desc = to_stm32_dma_desc(vdesc); chan->next_sg = 0; } @@ -555,6 +549,7 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) sg_req = &chan->desc->sg_req[chan->next_sg]; reg = &sg_req->chan_reg; + reg->dma_scr &= ~STM32_DMA_SCR_EN; stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); stm32_dma_write(dmadev, STM32_DMA_SPAR(chan->id), reg->dma_spar); stm32_dma_write(dmadev, STM32_DMA_SM0AR(chan->id), reg->dma_sm0ar); @@ -622,7 +617,6 @@ static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan) } else { chan->busy = false; if (chan->next_sg == chan->desc->num_sgs) { - list_del(&chan->desc->vdesc.node); vchan_cookie_complete(&chan->desc->vdesc); chan->desc = NULL; } @@ -1275,6 +1269,7 @@ static int stm32_dma_probe(struct platform_device *pdev) struct dma_device *dd; const struct of_device_id *match; struct resource *res; + struct reset_control *rst; int i, ret; match = of_match_device(stm32_dma_of_match, &pdev->dev); @@ -1296,8 +1291,10 @@ static int stm32_dma_probe(struct platform_device *pdev) dmadev->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(dmadev->clk)) { - dev_err(&pdev->dev, "Error: Missing controller clock\n"); - return PTR_ERR(dmadev->clk); + ret = PTR_ERR(dmadev->clk); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Can't get clock\n"); + return ret; } ret = clk_prepare_enable(dmadev->clk); @@ -1309,13 +1306,19 @@ static int stm32_dma_probe(struct platform_device *pdev) dmadev->mem2mem = of_property_read_bool(pdev->dev.of_node, "st,mem2mem"); - dmadev->rst = devm_reset_control_get(&pdev->dev, NULL); - if (!IS_ERR(dmadev->rst)) { - reset_control_assert(dmadev->rst); + rst = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(rst)) { + ret = PTR_ERR(rst); + if (ret == -EPROBE_DEFER) + goto clk_free; + } else { + reset_control_assert(rst); udelay(2); - reset_control_deassert(dmadev->rst); + reset_control_deassert(rst); } + dma_set_max_seg_size(&pdev->dev, STM32_DMA_ALIGNED_MAX_DATA_ITEMS); + dma_cap_set(DMA_SLAVE, dd->cap_mask); dma_cap_set(DMA_PRIVATE, dd->cap_mask); dma_cap_set(DMA_CYCLIC, dd->cap_mask); @@ -1336,7 +1339,9 @@ static int stm32_dma_probe(struct platform_device *pdev) BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dd->copy_align = DMAENGINE_ALIGN_32_BYTES; dd->max_burst = STM32_DMA_MAX_BURST; + dd->descriptor_reuse = true; dd->dev = &pdev->dev; INIT_LIST_HEAD(&dd->channels); @@ -1427,7 +1432,39 @@ static int stm32_dma_runtime_resume(struct device *dev) } #endif +#ifdef CONFIG_PM_SLEEP +static int stm32_dma_suspend(struct device *dev) +{ + struct stm32_dma_device *dmadev = dev_get_drvdata(dev); + int id, ret, scr; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) + return ret; + + for (id = 0; id < STM32_DMA_MAX_CHANNELS; id++) { + scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); + if (scr & STM32_DMA_SCR_EN) { + dev_warn(dev, "Suspend is prevented by Chan %i\n", id); + return -EBUSY; + } + } + + pm_runtime_put_sync(dev); + + pm_runtime_force_suspend(dev); + + return 0; +} + +static int stm32_dma_resume(struct device *dev) +{ + return pm_runtime_force_resume(dev); +} +#endif + static const struct dev_pm_ops stm32_dma_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(stm32_dma_suspend, stm32_dma_resume) SET_RUNTIME_PM_OPS(stm32_dma_runtime_suspend, stm32_dma_runtime_resume, NULL) }; @@ -1438,10 +1475,11 @@ static struct platform_driver stm32_dma_driver = { .of_match_table = stm32_dma_of_match, .pm = &stm32_dma_pm_ops, }, + .probe = stm32_dma_probe, }; static int __init stm32_dma_init(void) { - return platform_driver_probe(&stm32_dma_driver, stm32_dma_probe); + return platform_driver_register(&stm32_dma_driver); } subsys_initcall(stm32_dma_init); diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c index 3c89bd39e096..12f7637e13a1 100644 --- a/drivers/dma/stm32-dmamux.c +++ b/drivers/dma/stm32-dmamux.c @@ -35,12 +35,14 @@ struct stm32_dmamux { struct stm32_dmamux_data { struct dma_router dmarouter; struct clk *clk; - struct reset_control *rst; void __iomem *iomem; u32 dma_requests; /* Number of DMA requests connected to DMAMUX */ u32 dmamux_requests; /* Number of DMA requests routed toward DMAs */ spinlock_t lock; /* Protects register access */ unsigned long *dma_inuse; /* Used DMA channel */ + u32 ccr[STM32_DMAMUX_MAX_DMA_REQUESTS]; /* Used to backup CCR register + * in suspend + */ u32 dma_reqs[]; /* Number of DMA Request per DMA masters. * [0] holds number of DMA Masters. * To be kept at very end end of this structure @@ -179,6 +181,7 @@ static int stm32_dmamux_probe(struct platform_device *pdev) struct stm32_dmamux_data *stm32_dmamux; struct resource *res; void __iomem *iomem; + struct reset_control *rst; int i, count, ret; u32 dma_req; @@ -251,16 +254,26 @@ static int stm32_dmamux_probe(struct platform_device *pdev) stm32_dmamux->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(stm32_dmamux->clk)) { ret = PTR_ERR(stm32_dmamux->clk); - if (ret == -EPROBE_DEFER) - dev_info(&pdev->dev, "Missing controller clock\n"); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Missing clock controller\n"); return ret; } - stm32_dmamux->rst = devm_reset_control_get(&pdev->dev, NULL); - if (!IS_ERR(stm32_dmamux->rst)) { - reset_control_assert(stm32_dmamux->rst); + ret = clk_prepare_enable(stm32_dmamux->clk); + if (ret < 0) { + dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); + return ret; + } + + rst = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(rst)) { + ret = PTR_ERR(rst); + if (ret == -EPROBE_DEFER) + goto err_clk; + } else { + reset_control_assert(rst); udelay(2); - reset_control_deassert(stm32_dmamux->rst); + reset_control_deassert(rst); } stm32_dmamux->iomem = iomem; @@ -271,14 +284,6 @@ static int stm32_dmamux_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - if (!IS_ERR(stm32_dmamux->clk)) { - ret = clk_prepare_enable(stm32_dmamux->clk); - if (ret < 0) { - dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); - return ret; - } - } - pm_runtime_get_noresume(&pdev->dev); /* Reset the dmamux */ @@ -287,8 +292,17 @@ static int stm32_dmamux_probe(struct platform_device *pdev) pm_runtime_put(&pdev->dev); - return of_dma_router_register(node, stm32_dmamux_route_allocate, + ret = of_dma_router_register(node, stm32_dmamux_route_allocate, &stm32_dmamux->dmarouter); + if (ret) + goto err_clk; + + return 0; + +err_clk: + clk_disable_unprepare(stm32_dmamux->clk); + + return ret; } #ifdef CONFIG_PM @@ -318,7 +332,54 @@ static int stm32_dmamux_runtime_resume(struct device *dev) } #endif +#ifdef CONFIG_PM_SLEEP +static int stm32_dmamux_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); + int i, ret; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) + return ret; + + for (i = 0; i < stm32_dmamux->dma_requests; i++) + stm32_dmamux->ccr[i] = stm32_dmamux_read(stm32_dmamux->iomem, + STM32_DMAMUX_CCR(i)); + + pm_runtime_put_sync(dev); + + pm_runtime_force_suspend(dev); + + return 0; +} + +static int stm32_dmamux_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); + int i, ret; + + ret = pm_runtime_force_resume(dev); + if (ret < 0) + return ret; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) + return ret; + + for (i = 0; i < stm32_dmamux->dma_requests; i++) + stm32_dmamux_write(stm32_dmamux->iomem, STM32_DMAMUX_CCR(i), + stm32_dmamux->ccr[i]); + + pm_runtime_put_sync(dev); + + return 0; +} +#endif + static const struct dev_pm_ops stm32_dmamux_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(stm32_dmamux_suspend, stm32_dmamux_resume) SET_RUNTIME_PM_OPS(stm32_dmamux_runtime_suspend, stm32_dmamux_runtime_resume, NULL) }; diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c index 5838311cf990..5469563703d1 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32-mdma.c @@ -273,7 +273,6 @@ struct stm32_mdma_device { void __iomem *base; struct clk *clk; int irq; - struct reset_control *rst; u32 nr_channels; u32 nr_requests; u32 nr_ahb_addr_masks; @@ -1127,6 +1126,8 @@ static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan) return; } + list_del(&vdesc->node); + chan->desc = to_stm32_mdma_desc(vdesc); hwdesc = chan->desc->node[0].hwdesc; chan->curr_hwdesc = 0; @@ -1242,8 +1243,10 @@ static int stm32_mdma_terminate_all(struct dma_chan *c) LIST_HEAD(head); spin_lock_irqsave(&chan->vchan.lock, flags); - if (chan->busy) { - stm32_mdma_stop(chan); + if (chan->desc) { + vchan_terminate_vdesc(&chan->desc->vdesc); + if (chan->busy) + stm32_mdma_stop(chan); chan->desc = NULL; } vchan_get_all_descriptors(&chan->vchan, &head); @@ -1331,7 +1334,6 @@ static enum dma_status stm32_mdma_tx_status(struct dma_chan *c, static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan) { - list_del(&chan->desc->vdesc.node); vchan_cookie_complete(&chan->desc->vdesc); chan->desc = NULL; chan->busy = false; @@ -1532,6 +1534,7 @@ static int stm32_mdma_probe(struct platform_device *pdev) struct dma_device *dd; struct device_node *of_node; struct resource *res; + struct reset_control *rst; u32 nr_channels, nr_requests; int i, count, ret; @@ -1579,8 +1582,8 @@ static int stm32_mdma_probe(struct platform_device *pdev) dmadev->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(dmadev->clk)) { ret = PTR_ERR(dmadev->clk); - if (ret == -EPROBE_DEFER) - dev_info(&pdev->dev, "Missing controller clock\n"); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Missing clock controller\n"); return ret; } @@ -1590,11 +1593,15 @@ static int stm32_mdma_probe(struct platform_device *pdev) return ret; } - dmadev->rst = devm_reset_control_get(&pdev->dev, NULL); - if (!IS_ERR(dmadev->rst)) { - reset_control_assert(dmadev->rst); + rst = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(rst)) { + ret = PTR_ERR(rst); + if (ret == -EPROBE_DEFER) + goto err_clk; + } else { + reset_control_assert(rst); udelay(2); - reset_control_deassert(dmadev->rst); + reset_control_deassert(rst); } dd = &dmadev->ddev; @@ -1614,6 +1621,8 @@ static int stm32_mdma_probe(struct platform_device *pdev) dd->device_resume = stm32_mdma_resume; dd->device_terminate_all = stm32_mdma_terminate_all; dd->device_synchronize = stm32_mdma_synchronize; + dd->descriptor_reuse = true; + dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | @@ -1637,25 +1646,27 @@ static int stm32_mdma_probe(struct platform_device *pdev) } dmadev->irq = platform_get_irq(pdev, 0); - if (dmadev->irq < 0) - return dmadev->irq; + if (dmadev->irq < 0) { + ret = dmadev->irq; + goto err_clk; + } ret = devm_request_irq(&pdev->dev, dmadev->irq, stm32_mdma_irq_handler, 0, dev_name(&pdev->dev), dmadev); if (ret) { dev_err(&pdev->dev, "failed to request IRQ\n"); - return ret; + goto err_clk; } ret = dmaenginem_async_device_register(dd); if (ret) - return ret; + goto err_clk; ret = of_dma_controller_register(of_node, stm32_mdma_of_xlate, dmadev); if (ret < 0) { dev_err(&pdev->dev, "STM32 MDMA DMA OF registration failed %d\n", ret); - goto err_unregister; + goto err_clk; } platform_set_drvdata(pdev, dmadev); @@ -1668,7 +1679,9 @@ static int stm32_mdma_probe(struct platform_device *pdev) return 0; -err_unregister: +err_clk: + clk_disable_unprepare(dmadev->clk); + return ret; } @@ -1697,7 +1710,40 @@ static int stm32_mdma_runtime_resume(struct device *dev) } #endif +#ifdef CONFIG_PM_SLEEP +static int stm32_mdma_pm_suspend(struct device *dev) +{ + struct stm32_mdma_device *dmadev = dev_get_drvdata(dev); + u32 ccr, id; + int ret; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) + return ret; + + for (id = 0; id < dmadev->nr_channels; id++) { + ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id)); + if (ccr & STM32_MDMA_CCR_EN) { + dev_warn(dev, "Suspend is prevented by Chan %i\n", id); + return -EBUSY; + } + } + + pm_runtime_put_sync(dev); + + pm_runtime_force_suspend(dev); + + return 0; +} + +static int stm32_mdma_pm_resume(struct device *dev) +{ + return pm_runtime_force_resume(dev); +} +#endif + static const struct dev_pm_ops stm32_mdma_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(stm32_mdma_pm_suspend, stm32_mdma_pm_resume) SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend, stm32_mdma_runtime_resume, NULL) }; diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index bbc2bda3b902..e7ff09a5031d 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -697,11 +697,13 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, dest = sconfig->dst_addr; endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) | SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) | - SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type); + SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) | + SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode); } else { src = sconfig->src_addr; dest = buf; endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) | + SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) | SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) | SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode); } diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index 4a750e29bfb5..265303a396ca 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "dmaengine.h" @@ -59,7 +60,7 @@ #define TEGRA_APBDMA_STATUS_COUNT_MASK 0xFFFC #define TEGRA_APBDMA_CHAN_CSRE 0x00C -#define TEGRA_APBDMA_CHAN_CSRE_PAUSE (1 << 31) +#define TEGRA_APBDMA_CHAN_CSRE_PAUSE BIT(31) /* AHB memory address */ #define TEGRA_APBDMA_CHAN_AHBPTR 0x010 @@ -120,21 +121,21 @@ struct tegra_dma; * @support_separate_wcount_reg: Support separate word count register. */ struct tegra_dma_chip_data { - int nr_channels; - int channel_reg_size; - int max_dma_count; + unsigned int nr_channels; + unsigned int channel_reg_size; + unsigned int max_dma_count; bool support_channel_pause; bool support_separate_wcount_reg; }; /* DMA channel registers */ struct tegra_dma_channel_regs { - unsigned long csr; - unsigned long ahb_ptr; - unsigned long apb_ptr; - unsigned long ahb_seq; - unsigned long apb_seq; - unsigned long wcount; + u32 csr; + u32 ahb_ptr; + u32 apb_ptr; + u32 ahb_seq; + u32 apb_seq; + u32 wcount; }; /* @@ -168,7 +169,7 @@ struct tegra_dma_desc { struct list_head node; struct list_head tx_list; struct list_head cb_node; - int cb_count; + unsigned int cb_count; }; struct tegra_dma_channel; @@ -181,8 +182,7 @@ struct tegra_dma_channel { struct dma_chan dma_chan; char name[12]; bool config_init; - int id; - int irq; + unsigned int id; void __iomem *chan_addr; spinlock_t lock; bool busy; @@ -202,7 +202,9 @@ struct tegra_dma_channel { /* Channel-slave specific configuration */ unsigned int slave_id; struct dma_slave_config dma_sconfig; - struct tegra_dma_channel_regs channel_reg; + struct tegra_dma_channel_regs channel_reg; + + struct wait_queue_head wq; }; /* tegra_dma: Tegra DMA specific information */ @@ -222,9 +224,6 @@ struct tegra_dma { */ u32 global_pause_count; - /* Some register need to be cache before suspend */ - u32 reg_gen; - /* Last member of the structure */ struct tegra_dma_channel channels[0]; }; @@ -240,7 +239,7 @@ static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg) } static inline void tdc_write(struct tegra_dma_channel *tdc, - u32 reg, u32 val) + u32 reg, u32 val) { writel(val, tdc->chan_addr + reg); } @@ -255,8 +254,8 @@ static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) return container_of(dc, struct tegra_dma_channel, dma_chan); } -static inline struct tegra_dma_desc *txd_to_tegra_dma_desc( - struct dma_async_tx_descriptor *td) +static inline struct tegra_dma_desc * +txd_to_tegra_dma_desc(struct dma_async_tx_descriptor *td) { return container_of(td, struct tegra_dma_desc, txd); } @@ -267,12 +266,9 @@ static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) } static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx); -static int tegra_dma_runtime_suspend(struct device *dev); -static int tegra_dma_runtime_resume(struct device *dev); /* Get DMA desc from free list, if not there then allocate it. */ -static struct tegra_dma_desc *tegra_dma_desc_get( - struct tegra_dma_channel *tdc) +static struct tegra_dma_desc *tegra_dma_desc_get(struct tegra_dma_channel *tdc) { struct tegra_dma_desc *dma_desc; unsigned long flags; @@ -299,11 +295,12 @@ static struct tegra_dma_desc *tegra_dma_desc_get( dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan); dma_desc->txd.tx_submit = tegra_dma_tx_submit; dma_desc->txd.flags = 0; + return dma_desc; } static void tegra_dma_desc_put(struct tegra_dma_channel *tdc, - struct tegra_dma_desc *dma_desc) + struct tegra_dma_desc *dma_desc) { unsigned long flags; @@ -314,29 +311,29 @@ static void tegra_dma_desc_put(struct tegra_dma_channel *tdc, spin_unlock_irqrestore(&tdc->lock, flags); } -static struct tegra_dma_sg_req *tegra_dma_sg_req_get( - struct tegra_dma_channel *tdc) +static struct tegra_dma_sg_req * +tegra_dma_sg_req_get(struct tegra_dma_channel *tdc) { - struct tegra_dma_sg_req *sg_req = NULL; + struct tegra_dma_sg_req *sg_req; unsigned long flags; spin_lock_irqsave(&tdc->lock, flags); if (!list_empty(&tdc->free_sg_req)) { - sg_req = list_first_entry(&tdc->free_sg_req, - typeof(*sg_req), node); + sg_req = list_first_entry(&tdc->free_sg_req, typeof(*sg_req), + node); list_del(&sg_req->node); spin_unlock_irqrestore(&tdc->lock, flags); return sg_req; } spin_unlock_irqrestore(&tdc->lock, flags); - sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_NOWAIT); + sg_req = kzalloc(sizeof(*sg_req), GFP_NOWAIT); return sg_req; } static int tegra_dma_slave_config(struct dma_chan *dc, - struct dma_slave_config *sconfig) + struct dma_slave_config *sconfig) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); @@ -353,11 +350,12 @@ static int tegra_dma_slave_config(struct dma_chan *dc, tdc->slave_id = sconfig->slave_id; } tdc->config_init = true; + return 0; } static void tegra_dma_global_pause(struct tegra_dma_channel *tdc, - bool wait_for_burst_complete) + bool wait_for_burst_complete) { struct tegra_dma *tdma = tdc->tdma; @@ -392,13 +390,13 @@ static void tegra_dma_global_resume(struct tegra_dma_channel *tdc) } static void tegra_dma_pause(struct tegra_dma_channel *tdc, - bool wait_for_burst_complete) + bool wait_for_burst_complete) { struct tegra_dma *tdma = tdc->tdma; if (tdma->chip_data->support_channel_pause) { tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, - TEGRA_APBDMA_CHAN_CSRE_PAUSE); + TEGRA_APBDMA_CHAN_CSRE_PAUSE); if (wait_for_burst_complete) udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); } else { @@ -410,17 +408,15 @@ static void tegra_dma_resume(struct tegra_dma_channel *tdc) { struct tegra_dma *tdma = tdc->tdma; - if (tdma->chip_data->support_channel_pause) { + if (tdma->chip_data->support_channel_pause) tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0); - } else { + else tegra_dma_global_resume(tdc); - } } static void tegra_dma_stop(struct tegra_dma_channel *tdc) { - u32 csr; - u32 status; + u32 csr, status; /* Disable interrupts */ csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); @@ -441,7 +437,7 @@ static void tegra_dma_stop(struct tegra_dma_channel *tdc) } static void tegra_dma_start(struct tegra_dma_channel *tdc, - struct tegra_dma_sg_req *sg_req) + struct tegra_dma_sg_req *sg_req) { struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs; @@ -455,11 +451,11 @@ static void tegra_dma_start(struct tegra_dma_channel *tdc, /* Start DMA */ tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, - ch_regs->csr | TEGRA_APBDMA_CSR_ENB); + ch_regs->csr | TEGRA_APBDMA_CSR_ENB); } static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, - struct tegra_dma_sg_req *nsg_req) + struct tegra_dma_sg_req *nsg_req) { unsigned long status; @@ -493,9 +489,9 @@ static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr); if (tdc->tdma->chip_data->support_separate_wcount_reg) tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, - nsg_req->ch_regs.wcount); + nsg_req->ch_regs.wcount); tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, - nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); + nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); nsg_req->configured = true; nsg_req->words_xferred = 0; @@ -506,11 +502,7 @@ static void tdc_start_head_req(struct tegra_dma_channel *tdc) { struct tegra_dma_sg_req *sg_req; - if (list_empty(&tdc->pending_sg_req)) - return; - - sg_req = list_first_entry(&tdc->pending_sg_req, - typeof(*sg_req), node); + sg_req = list_first_entry(&tdc->pending_sg_req, typeof(*sg_req), node); tegra_dma_start(tdc, sg_req); sg_req->configured = true; sg_req->words_xferred = 0; @@ -519,34 +511,32 @@ static void tdc_start_head_req(struct tegra_dma_channel *tdc) static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc) { - struct tegra_dma_sg_req *hsgreq; - struct tegra_dma_sg_req *hnsgreq; - - if (list_empty(&tdc->pending_sg_req)) - return; + struct tegra_dma_sg_req *hsgreq, *hnsgreq; hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) { - hnsgreq = list_first_entry(&hsgreq->node, - typeof(*hnsgreq), node); + hnsgreq = list_first_entry(&hsgreq->node, typeof(*hnsgreq), + node); tegra_dma_configure_for_next(tdc, hnsgreq); } } -static inline int get_current_xferred_count(struct tegra_dma_channel *tdc, - struct tegra_dma_sg_req *sg_req, unsigned long status) +static inline unsigned int +get_current_xferred_count(struct tegra_dma_channel *tdc, + struct tegra_dma_sg_req *sg_req, + unsigned long status) { return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4; } static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) { - struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; + struct tegra_dma_sg_req *sgreq; while (!list_empty(&tdc->pending_sg_req)) { - sgreq = list_first_entry(&tdc->pending_sg_req, - typeof(*sgreq), node); + sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), + node); list_move_tail(&sgreq->node, &tdc->free_sg_req); if (sgreq->last_sg) { dma_desc = sgreq->dma_desc; @@ -556,7 +546,7 @@ static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) /* Add in cb list if it is not there. */ if (!dma_desc->cb_count) list_add_tail(&dma_desc->cb_node, - &tdc->cb_desc); + &tdc->cb_desc); dma_desc->cb_count++; } } @@ -564,15 +554,9 @@ static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) } static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, - struct tegra_dma_sg_req *last_sg_req, bool to_terminate) + bool to_terminate) { - struct tegra_dma_sg_req *hsgreq = NULL; - - if (list_empty(&tdc->pending_sg_req)) { - dev_err(tdc2dev(tdc), "DMA is running without req\n"); - tegra_dma_stop(tdc); - return false; - } + struct tegra_dma_sg_req *hsgreq; /* * Check that head req on list should be in flight. @@ -582,7 +566,8 @@ static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); if (!hsgreq->configured) { tegra_dma_stop(tdc); - dev_err(tdc2dev(tdc), "Error in DMA transfer, aborting DMA\n"); + pm_runtime_put(tdc->tdma->dev); + dev_err(tdc2dev(tdc), "DMA transfer underflow, aborting DMA\n"); tegra_dma_abort_all(tdc); return false; } @@ -590,14 +575,15 @@ static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, /* Configure next request */ if (!to_terminate) tdc_configure_next_head_desc(tdc); + return true; } static void handle_once_dma_done(struct tegra_dma_channel *tdc, - bool to_terminate) + bool to_terminate) { - struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; + struct tegra_dma_sg_req *sgreq; tdc->busy = false; sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); @@ -616,17 +602,22 @@ static void handle_once_dma_done(struct tegra_dma_channel *tdc, list_add_tail(&sgreq->node, &tdc->free_sg_req); /* Do not start DMA if it is going to be terminate */ - if (to_terminate || list_empty(&tdc->pending_sg_req)) + if (to_terminate) return; + if (list_empty(&tdc->pending_sg_req)) { + pm_runtime_put(tdc->tdma->dev); + return; + } + tdc_start_head_req(tdc); } static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, - bool to_terminate) + bool to_terminate) { - struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; + struct tegra_dma_sg_req *sgreq; bool st; sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); @@ -647,7 +638,7 @@ static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) { list_move_tail(&sgreq->node, &tdc->pending_sg_req); sgreq->configured = false; - st = handle_continuous_head_request(tdc, sgreq, to_terminate); + st = handle_continuous_head_request(tdc, to_terminate); if (!st) dma_desc->dma_status = DMA_ERROR; } @@ -658,13 +649,13 @@ static void tegra_dma_tasklet(unsigned long data) struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data; struct dmaengine_desc_callback cb; struct tegra_dma_desc *dma_desc; + unsigned int cb_count; unsigned long flags; - int cb_count; spin_lock_irqsave(&tdc->lock, flags); while (!list_empty(&tdc->cb_desc)) { - dma_desc = list_first_entry(&tdc->cb_desc, - typeof(*dma_desc), cb_node); + dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), + cb_node); list_del(&dma_desc->cb_node); dmaengine_desc_get_callback(&dma_desc->txd, &cb); cb_count = dma_desc->cb_count; @@ -682,10 +673,9 @@ static void tegra_dma_tasklet(unsigned long data) static irqreturn_t tegra_dma_isr(int irq, void *dev_id) { struct tegra_dma_channel *tdc = dev_id; - unsigned long status; - unsigned long flags; + u32 status; - spin_lock_irqsave(&tdc->lock, flags); + spin_lock(&tdc->lock); trace_tegra_dma_isr(&tdc->dma_chan, irq); status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); @@ -693,13 +683,15 @@ static irqreturn_t tegra_dma_isr(int irq, void *dev_id) tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); tdc->isr_handler(tdc, false); tasklet_schedule(&tdc->tasklet); - spin_unlock_irqrestore(&tdc->lock, flags); + wake_up_all(&tdc->wq); + spin_unlock(&tdc->lock); return IRQ_HANDLED; } - spin_unlock_irqrestore(&tdc->lock, flags); - dev_info(tdc2dev(tdc), - "Interrupt already served status 0x%08lx\n", status); + spin_unlock(&tdc->lock); + dev_info(tdc2dev(tdc), "Interrupt already served status 0x%08x\n", + status); + return IRQ_NONE; } @@ -715,6 +707,7 @@ static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *txd) cookie = dma_cookie_assign(&dma_desc->txd); list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req); spin_unlock_irqrestore(&tdc->lock, flags); + return cookie; } @@ -722,6 +715,7 @@ static void tegra_dma_issue_pending(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); unsigned long flags; + int err; spin_lock_irqsave(&tdc->lock, flags); if (list_empty(&tdc->pending_sg_req)) { @@ -729,6 +723,12 @@ static void tegra_dma_issue_pending(struct dma_chan *dc) goto end; } if (!tdc->busy) { + err = pm_runtime_get_sync(tdc->tdma->dev); + if (err < 0) { + dev_err(tdc2dev(tdc), "Failed to enable DMA\n"); + goto end; + } + tdc_start_head_req(tdc); /* Continuous single mode: Configure next req */ @@ -748,11 +748,10 @@ static void tegra_dma_issue_pending(struct dma_chan *dc) static int tegra_dma_terminate_all(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); - struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; + struct tegra_dma_sg_req *sgreq; unsigned long flags; - unsigned long status; - unsigned long wcount; + u32 status, wcount; bool was_busy; spin_lock_irqsave(&tdc->lock, flags); @@ -778,30 +777,60 @@ static int tegra_dma_terminate_all(struct dma_chan *dc) tegra_dma_stop(tdc); if (!list_empty(&tdc->pending_sg_req) && was_busy) { - sgreq = list_first_entry(&tdc->pending_sg_req, - typeof(*sgreq), node); + sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), + node); sgreq->dma_desc->bytes_transferred += get_current_xferred_count(tdc, sgreq, wcount); } tegra_dma_resume(tdc); + pm_runtime_put(tdc->tdma->dev); + wake_up_all(&tdc->wq); + skip_dma_stop: tegra_dma_abort_all(tdc); while (!list_empty(&tdc->cb_desc)) { - dma_desc = list_first_entry(&tdc->cb_desc, - typeof(*dma_desc), cb_node); + dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), + cb_node); list_del(&dma_desc->cb_node); dma_desc->cb_count = 0; } spin_unlock_irqrestore(&tdc->lock, flags); + return 0; } +static bool tegra_dma_eoc_interrupt_deasserted(struct tegra_dma_channel *tdc) +{ + unsigned long flags; + u32 status; + + spin_lock_irqsave(&tdc->lock, flags); + status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); + spin_unlock_irqrestore(&tdc->lock, flags); + + return !(status & TEGRA_APBDMA_STATUS_ISE_EOC); +} + +static void tegra_dma_synchronize(struct dma_chan *dc) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + + /* + * CPU, which handles interrupt, could be busy in + * uninterruptible state, in this case sibling CPU + * should wait until interrupt is handled. + */ + wait_event(tdc->wq, tegra_dma_eoc_interrupt_deasserted(tdc)); + + tasklet_kill(&tdc->tasklet); +} + static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc, struct tegra_dma_sg_req *sg_req) { - unsigned long status, wcount = 0; + u32 status, wcount = 0; if (!list_is_first(&sg_req->node, &tdc->pending_sg_req)) return 0; @@ -858,7 +887,8 @@ static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc, } static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, - dma_cookie_t cookie, struct dma_tx_state *txstate) + dma_cookie_t cookie, + struct dma_tx_state *txstate) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_desc *dma_desc; @@ -905,11 +935,12 @@ static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, trace_tegra_dma_tx_status(&tdc->dma_chan, cookie, txstate); spin_unlock_irqrestore(&tdc->lock, flags); + return ret; } -static inline int get_bus_width(struct tegra_dma_channel *tdc, - enum dma_slave_buswidth slave_bw) +static inline unsigned int get_bus_width(struct tegra_dma_channel *tdc, + enum dma_slave_buswidth slave_bw) { switch (slave_bw) { case DMA_SLAVE_BUSWIDTH_1_BYTE: @@ -922,16 +953,17 @@ static inline int get_bus_width(struct tegra_dma_channel *tdc, return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64; default: dev_warn(tdc2dev(tdc), - "slave bw is not supported, using 32bits\n"); + "slave bw is not supported, using 32bits\n"); return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32; } } -static inline int get_burst_size(struct tegra_dma_channel *tdc, - u32 burst_size, enum dma_slave_buswidth slave_bw, int len) +static inline unsigned int get_burst_size(struct tegra_dma_channel *tdc, + u32 burst_size, + enum dma_slave_buswidth slave_bw, + u32 len) { - int burst_byte; - int burst_ahb_width; + unsigned int burst_byte, burst_ahb_width; /* * burst_size from client is in terms of the bus_width. @@ -958,9 +990,12 @@ static inline int get_burst_size(struct tegra_dma_channel *tdc, } static int get_transfer_param(struct tegra_dma_channel *tdc, - enum dma_transfer_direction direction, unsigned long *apb_addr, - unsigned long *apb_seq, unsigned long *csr, unsigned int *burst_size, - enum dma_slave_buswidth *slave_bw) + enum dma_transfer_direction direction, + u32 *apb_addr, + u32 *apb_seq, + u32 *csr, + unsigned int *burst_size, + enum dma_slave_buswidth *slave_bw) { switch (direction) { case DMA_MEM_TO_DEV: @@ -981,13 +1016,15 @@ static int get_transfer_param(struct tegra_dma_channel *tdc, default: dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); - return -EINVAL; + break; } + return -EINVAL; } static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc, - struct tegra_dma_channel_regs *ch_regs, u32 len) + struct tegra_dma_channel_regs *ch_regs, + u32 len) { u32 len_field = (len - 4) & 0xFFFC; @@ -997,20 +1034,23 @@ static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc, ch_regs->csr |= len_field; } -static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( - struct dma_chan *dc, struct scatterlist *sgl, unsigned int sg_len, - enum dma_transfer_direction direction, unsigned long flags, - void *context) +static struct dma_async_tx_descriptor * +tegra_dma_prep_slave_sg(struct dma_chan *dc, + struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, + void *context) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); - struct tegra_dma_desc *dma_desc; - unsigned int i; - struct scatterlist *sg; - unsigned long csr, ahb_seq, apb_ptr, apb_seq; - struct list_head req_list; - struct tegra_dma_sg_req *sg_req = NULL; - u32 burst_size; + struct tegra_dma_sg_req *sg_req = NULL; + u32 csr, ahb_seq, apb_ptr, apb_seq; enum dma_slave_buswidth slave_bw; + struct tegra_dma_desc *dma_desc; + struct list_head req_list; + struct scatterlist *sg; + unsigned int burst_size; + unsigned int i; if (!tdc->config_init) { dev_err(tdc2dev(tdc), "DMA channel is not configured\n"); @@ -1022,7 +1062,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( } if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, - &burst_size, &slave_bw) < 0) + &burst_size, &slave_bw) < 0) return NULL; INIT_LIST_HEAD(&req_list); @@ -1068,7 +1108,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( len = sg_dma_len(sg); if ((len & 3) || (mem & 3) || - (len > tdc->tdma->chip_data->max_dma_count)) { + len > tdc->tdma->chip_data->max_dma_count) { dev_err(tdc2dev(tdc), "DMA length/memory address is not supported\n"); tegra_dma_desc_put(tdc, dma_desc); @@ -1120,20 +1160,21 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( return &dma_desc->txd; } -static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( - struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, - size_t period_len, enum dma_transfer_direction direction, - unsigned long flags) +static struct dma_async_tx_descriptor * +tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, + size_t buf_len, + size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); - struct tegra_dma_desc *dma_desc = NULL; struct tegra_dma_sg_req *sg_req = NULL; - unsigned long csr, ahb_seq, apb_ptr, apb_seq; - int len; - size_t remain_len; - dma_addr_t mem = buf_addr; - u32 burst_size; + u32 csr, ahb_seq, apb_ptr, apb_seq; enum dma_slave_buswidth slave_bw; + struct tegra_dma_desc *dma_desc; + dma_addr_t mem = buf_addr; + unsigned int burst_size; + size_t len, remain_len; if (!buf_len || !period_len) { dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); @@ -1167,13 +1208,13 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( len = period_len; if ((len & 3) || (buf_addr & 3) || - (len > tdc->tdma->chip_data->max_dma_count)) { + len > tdc->tdma->chip_data->max_dma_count) { dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n"); return NULL; } if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, - &burst_size, &slave_bw) < 0) + &burst_size, &slave_bw) < 0) return NULL; ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; @@ -1259,15 +1300,8 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); - struct tegra_dma *tdma = tdc->tdma; - int ret; dma_cookie_init(&tdc->dma_chan); - tdc->config_init = false; - - ret = pm_runtime_get_sync(tdma->dev); - if (ret < 0) - return ret; return 0; } @@ -1275,33 +1309,29 @@ static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) static void tegra_dma_free_chan_resources(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); - struct tegra_dma *tdma = tdc->tdma; struct tegra_dma_desc *dma_desc; struct tegra_dma_sg_req *sg_req; struct list_head dma_desc_list; struct list_head sg_req_list; - unsigned long flags; INIT_LIST_HEAD(&dma_desc_list); INIT_LIST_HEAD(&sg_req_list); dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id); - if (tdc->busy) - tegra_dma_terminate_all(dc); + tegra_dma_terminate_all(dc); + tasklet_kill(&tdc->tasklet); - spin_lock_irqsave(&tdc->lock, flags); list_splice_init(&tdc->pending_sg_req, &sg_req_list); list_splice_init(&tdc->free_sg_req, &sg_req_list); list_splice_init(&tdc->free_dma_desc, &dma_desc_list); INIT_LIST_HEAD(&tdc->cb_desc); tdc->config_init = false; tdc->isr_handler = NULL; - spin_unlock_irqrestore(&tdc->lock, flags); while (!list_empty(&dma_desc_list)) { - dma_desc = list_first_entry(&dma_desc_list, - typeof(*dma_desc), node); + dma_desc = list_first_entry(&dma_desc_list, typeof(*dma_desc), + node); list_del(&dma_desc->node); kfree(dma_desc); } @@ -1311,7 +1341,6 @@ static void tegra_dma_free_chan_resources(struct dma_chan *dc) list_del(&sg_req->node); kfree(sg_req); } - pm_runtime_put(tdma->dev); tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; } @@ -1320,8 +1349,8 @@ static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct tegra_dma *tdma = ofdma->of_dma_data; - struct dma_chan *chan; struct tegra_dma_channel *tdc; + struct dma_chan *chan; if (dma_spec->args[0] > TEGRA_APBDMA_CSR_REQ_SEL_MASK) { dev_err(tdma->dev, "Invalid slave id: %d\n", dma_spec->args[0]); @@ -1374,23 +1403,48 @@ static const struct tegra_dma_chip_data tegra148_dma_chip_data = { .support_separate_wcount_reg = true, }; -static int tegra_dma_probe(struct platform_device *pdev) +static int tegra_dma_init_hw(struct tegra_dma *tdma) { - struct resource *res; - struct tegra_dma *tdma; - int ret; - int i; - const struct tegra_dma_chip_data *cdata; + int err; - cdata = of_device_get_match_data(&pdev->dev); - if (!cdata) { - dev_err(&pdev->dev, "Error: No device match data found\n"); - return -ENODEV; + err = reset_control_assert(tdma->rst); + if (err) { + dev_err(tdma->dev, "failed to assert reset: %d\n", err); + return err; } - tdma = devm_kzalloc(&pdev->dev, - struct_size(tdma, channels, cdata->nr_channels), - GFP_KERNEL); + err = clk_enable(tdma->dma_clk); + if (err) { + dev_err(tdma->dev, "failed to enable clk: %d\n", err); + return err; + } + + /* reset DMA controller */ + udelay(2); + reset_control_deassert(tdma->rst); + + /* enable global DMA registers */ + tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); + tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); + tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFF); + + clk_disable(tdma->dma_clk); + + return 0; +} + +static int tegra_dma_probe(struct platform_device *pdev) +{ + const struct tegra_dma_chip_data *cdata; + struct tegra_dma *tdma; + unsigned int i; + size_t size; + int ret; + + cdata = of_device_get_match_data(&pdev->dev); + size = struct_size(tdma, channels, cdata->nr_channels); + + tdma = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); if (!tdma) return -ENOMEM; @@ -1398,8 +1452,7 @@ static int tegra_dma_probe(struct platform_device *pdev) tdma->chip_data = cdata; platform_set_drvdata(pdev, tdma); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - tdma->base_addr = devm_ioremap_resource(&pdev->dev, res); + tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(tdma->base_addr)) return PTR_ERR(tdma->base_addr); @@ -1417,64 +1470,55 @@ static int tegra_dma_probe(struct platform_device *pdev) spin_lock_init(&tdma->global_lock); - pm_runtime_enable(&pdev->dev); - if (!pm_runtime_enabled(&pdev->dev)) - ret = tegra_dma_runtime_resume(&pdev->dev); - else - ret = pm_runtime_get_sync(&pdev->dev); - - if (ret < 0) { - pm_runtime_disable(&pdev->dev); + ret = clk_prepare(tdma->dma_clk); + if (ret) return ret; - } - /* Reset DMA controller */ - reset_control_assert(tdma->rst); - udelay(2); - reset_control_deassert(tdma->rst); + ret = tegra_dma_init_hw(tdma); + if (ret) + goto err_clk_unprepare; - /* Enable global DMA registers */ - tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); - tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); - tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul); - - pm_runtime_put(&pdev->dev); + pm_runtime_irq_safe(&pdev->dev); + pm_runtime_enable(&pdev->dev); INIT_LIST_HEAD(&tdma->dma_dev.channels); for (i = 0; i < cdata->nr_channels; i++) { struct tegra_dma_channel *tdc = &tdma->channels[i]; + int irq; tdc->chan_addr = tdma->base_addr + TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + (i * cdata->channel_reg_size); - res = platform_get_resource(pdev, IORESOURCE_IRQ, i); - if (!res) { - ret = -EINVAL; + irq = platform_get_irq(pdev, i); + if (irq < 0) { + ret = irq; dev_err(&pdev->dev, "No irq resource for chan %d\n", i); - goto err_irq; + goto err_pm_disable; } - tdc->irq = res->start; + snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i); - ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc); + ret = devm_request_irq(&pdev->dev, irq, tegra_dma_isr, 0, + tdc->name, tdc); if (ret) { dev_err(&pdev->dev, "request_irq failed with err %d channel %d\n", ret, i); - goto err_irq; + goto err_pm_disable; } tdc->dma_chan.device = &tdma->dma_dev; dma_cookie_init(&tdc->dma_chan); list_add_tail(&tdc->dma_chan.device_node, - &tdma->dma_dev.channels); + &tdma->dma_dev.channels); tdc->tdma = tdma; tdc->id = i; tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; tasklet_init(&tdc->tasklet, tegra_dma_tasklet, - (unsigned long)tdc); + (unsigned long)tdc); spin_lock_init(&tdc->lock); + init_waitqueue_head(&tdc->wq); INIT_LIST_HEAD(&tdc->pending_sg_req); INIT_LIST_HEAD(&tdc->free_sg_req); @@ -1506,6 +1550,7 @@ static int tegra_dma_probe(struct platform_device *pdev) tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; tdma->dma_dev.device_config = tegra_dma_slave_config; tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all; + tdma->dma_dev.device_synchronize = tegra_dma_synchronize; tdma->dma_dev.device_tx_status = tegra_dma_tx_status; tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending; @@ -1513,7 +1558,7 @@ static int tegra_dma_probe(struct platform_device *pdev) if (ret < 0) { dev_err(&pdev->dev, "Tegra20 APB DMA driver registration failed %d\n", ret); - goto err_irq; + goto err_pm_disable; } ret = of_dma_controller_register(pdev->dev.of_node, @@ -1524,118 +1569,92 @@ static int tegra_dma_probe(struct platform_device *pdev) goto err_unregister_dma_dev; } - dev_info(&pdev->dev, "Tegra20 APB DMA driver register %d channels\n", - cdata->nr_channels); + dev_info(&pdev->dev, "Tegra20 APB DMA driver registered %u channels\n", + cdata->nr_channels); + return 0; err_unregister_dma_dev: dma_async_device_unregister(&tdma->dma_dev); -err_irq: - while (--i >= 0) { - struct tegra_dma_channel *tdc = &tdma->channels[i]; - - free_irq(tdc->irq, tdc); - tasklet_kill(&tdc->tasklet); - } +err_pm_disable: pm_runtime_disable(&pdev->dev); - if (!pm_runtime_status_suspended(&pdev->dev)) - tegra_dma_runtime_suspend(&pdev->dev); + +err_clk_unprepare: + clk_unprepare(tdma->dma_clk); + return ret; } static int tegra_dma_remove(struct platform_device *pdev) { struct tegra_dma *tdma = platform_get_drvdata(pdev); - int i; - struct tegra_dma_channel *tdc; + of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&tdma->dma_dev); - - for (i = 0; i < tdma->chip_data->nr_channels; ++i) { - tdc = &tdma->channels[i]; - free_irq(tdc->irq, tdc); - tasklet_kill(&tdc->tasklet); - } - pm_runtime_disable(&pdev->dev); - if (!pm_runtime_status_suspended(&pdev->dev)) - tegra_dma_runtime_suspend(&pdev->dev); + clk_unprepare(tdma->dma_clk); return 0; } -static int tegra_dma_runtime_suspend(struct device *dev) +static int __maybe_unused tegra_dma_runtime_suspend(struct device *dev) { struct tegra_dma *tdma = dev_get_drvdata(dev); - int i; - tdma->reg_gen = tdma_read(tdma, TEGRA_APBDMA_GENERAL); - for (i = 0; i < tdma->chip_data->nr_channels; i++) { - struct tegra_dma_channel *tdc = &tdma->channels[i]; - struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg; - - /* Only save the state of DMA channels that are in use */ - if (!tdc->config_init) - continue; - - ch_reg->csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); - ch_reg->ahb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBPTR); - ch_reg->apb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBPTR); - ch_reg->ahb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBSEQ); - ch_reg->apb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBSEQ); - if (tdma->chip_data->support_separate_wcount_reg) - ch_reg->wcount = tdc_read(tdc, - TEGRA_APBDMA_CHAN_WCOUNT); - } - - clk_disable_unprepare(tdma->dma_clk); + clk_disable(tdma->dma_clk); return 0; } -static int tegra_dma_runtime_resume(struct device *dev) +static int __maybe_unused tegra_dma_runtime_resume(struct device *dev) { struct tegra_dma *tdma = dev_get_drvdata(dev); - int i, ret; - ret = clk_prepare_enable(tdma->dma_clk); - if (ret < 0) { - dev_err(dev, "clk_enable failed: %d\n", ret); - return ret; - } + return clk_enable(tdma->dma_clk); +} - tdma_write(tdma, TEGRA_APBDMA_GENERAL, tdma->reg_gen); - tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); - tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul); +static int __maybe_unused tegra_dma_dev_suspend(struct device *dev) +{ + struct tegra_dma *tdma = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + bool busy; for (i = 0; i < tdma->chip_data->nr_channels; i++) { struct tegra_dma_channel *tdc = &tdma->channels[i]; - struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg; - /* Only restore the state of DMA channels that are in use */ - if (!tdc->config_init) - continue; + tasklet_kill(&tdc->tasklet); - if (tdma->chip_data->support_separate_wcount_reg) - tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, - ch_reg->wcount); - tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_reg->apb_seq); - tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_reg->apb_ptr); - tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_reg->ahb_seq); - tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_reg->ahb_ptr); - tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, - (ch_reg->csr & ~TEGRA_APBDMA_CSR_ENB)); + spin_lock_irqsave(&tdc->lock, flags); + busy = tdc->busy; + spin_unlock_irqrestore(&tdc->lock, flags); + + if (busy) { + dev_err(tdma->dev, "channel %u busy\n", i); + return -EBUSY; + } } - return 0; + return pm_runtime_force_suspend(dev); +} + +static int __maybe_unused tegra_dma_dev_resume(struct device *dev) +{ + struct tegra_dma *tdma = dev_get_drvdata(dev); + int err; + + err = tegra_dma_init_hw(tdma); + if (err) + return err; + + return pm_runtime_force_resume(dev); } static const struct dev_pm_ops tegra_dma_dev_pm_ops = { SET_RUNTIME_PM_OPS(tegra_dma_runtime_suspend, tegra_dma_runtime_resume, NULL) - SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) + SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_dev_suspend, tegra_dma_dev_resume) }; static const struct of_device_id tegra_dma_of_match[] = { @@ -1668,7 +1687,6 @@ static struct platform_driver tegra_dmac_driver = { module_platform_driver(tegra_dmac_driver); -MODULE_ALIAS("platform:tegra20-apbdma"); MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver"); MODULE_AUTHOR("Laxman Dewangan "); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index 6e1268552f74..c4ce5dfb149b 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -164,7 +164,7 @@ struct tegra_adma { const struct tegra_adma_chip_data *cdata; /* Last member of the structure */ - struct tegra_adma_chan channels[0]; + struct tegra_adma_chan channels[]; }; static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val) diff --git a/drivers/dma/ti/dma-crossbar.c b/drivers/dma/ti/dma-crossbar.c index f255056696ee..4ba8fa5d9c36 100644 --- a/drivers/dma/ti/dma-crossbar.c +++ b/drivers/dma/ti/dma-crossbar.c @@ -133,7 +133,6 @@ static int ti_am335x_xbar_probe(struct platform_device *pdev) const struct of_device_id *match; struct device_node *dma_node; struct ti_am335x_xbar_data *xbar; - struct resource *res; void __iomem *iomem; int i, ret; @@ -173,8 +172,7 @@ static int ti_am335x_xbar_probe(struct platform_device *pdev) xbar->xbar_events = TI_AM335X_XBAR_LINES; } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - iomem = devm_ioremap_resource(&pdev->dev, res); + iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(iomem)) return PTR_ERR(iomem); @@ -323,7 +321,6 @@ static int ti_dra7_xbar_probe(struct platform_device *pdev) struct device_node *dma_node; struct ti_dra7_xbar_data *xbar; struct property *prop; - struct resource *res; u32 safe_val; int sz; void __iomem *iomem; @@ -403,8 +400,7 @@ static int ti_dra7_xbar_probe(struct platform_device *pdev) kfree(rsv_events); } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - iomem = devm_ioremap_resource(&pdev->dev, res); + iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(iomem)) return PTR_ERR(iomem); diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c index 03a7f647f7b2..c4a5c170c1f9 100644 --- a/drivers/dma/ti/edma.c +++ b/drivers/dma/ti/edma.c @@ -1275,6 +1275,81 @@ static struct dma_async_tx_descriptor *edma_prep_dma_memcpy( return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); } +static struct dma_async_tx_descriptor * +edma_prep_dma_interleaved(struct dma_chan *chan, + struct dma_interleaved_template *xt, + unsigned long tx_flags) +{ + struct device *dev = chan->device->dev; + struct edma_chan *echan = to_edma_chan(chan); + struct edmacc_param *param; + struct edma_desc *edesc; + size_t src_icg, dst_icg; + int src_bidx, dst_bidx; + + /* Slave mode is not supported */ + if (is_slave_direction(xt->dir)) + return NULL; + + if (xt->frame_size != 1 || xt->numf == 0) + return NULL; + + if (xt->sgl[0].size > SZ_64K || xt->numf > SZ_64K) + return NULL; + + src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); + if (src_icg) { + src_bidx = src_icg + xt->sgl[0].size; + } else if (xt->src_inc) { + src_bidx = xt->sgl[0].size; + } else { + dev_err(dev, "%s: SRC constant addressing is not supported\n", + __func__); + return NULL; + } + + dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); + if (dst_icg) { + dst_bidx = dst_icg + xt->sgl[0].size; + } else if (xt->dst_inc) { + dst_bidx = xt->sgl[0].size; + } else { + dev_err(dev, "%s: DST constant addressing is not supported\n", + __func__); + return NULL; + } + + if (src_bidx > SZ_64K || dst_bidx > SZ_64K) + return NULL; + + edesc = kzalloc(struct_size(edesc, pset, 1), GFP_ATOMIC); + if (!edesc) + return NULL; + + edesc->direction = DMA_MEM_TO_MEM; + edesc->echan = echan; + edesc->pset_nr = 1; + + param = &edesc->pset[0].param; + + param->src = xt->src_start; + param->dst = xt->dst_start; + param->a_b_cnt = xt->numf << 16 | xt->sgl[0].size; + param->ccnt = 1; + param->src_dst_bidx = (dst_bidx << 16) | src_bidx; + param->src_dst_cidx = 0; + + param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num)); + param->opt |= ITCCHEN; + /* Enable transfer complete interrupt if requested */ + if (tx_flags & DMA_PREP_INTERRUPT) + param->opt |= TCINTEN; + else + edesc->polled = true; + + return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); +} + static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, @@ -1917,7 +1992,9 @@ static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode) "Legacy memcpy is enabled, things might not work\n"); dma_cap_set(DMA_MEMCPY, s_ddev->cap_mask); + dma_cap_set(DMA_INTERLEAVE, s_ddev->cap_mask); s_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy; + s_ddev->device_prep_interleaved_dma = edma_prep_dma_interleaved; s_ddev->directions = BIT(DMA_MEM_TO_MEM); } @@ -1953,8 +2030,10 @@ static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode) dma_cap_zero(m_ddev->cap_mask); dma_cap_set(DMA_MEMCPY, m_ddev->cap_mask); + dma_cap_set(DMA_INTERLEAVE, m_ddev->cap_mask); m_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy; + m_ddev->device_prep_interleaved_dma = edma_prep_dma_interleaved; m_ddev->device_alloc_chan_resources = edma_alloc_chan_resources; m_ddev->device_free_chan_resources = edma_free_chan_resources; m_ddev->device_issue_pending = edma_issue_pending; diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index 4d7561a1b3e3..64c8955e0cf1 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -32,6 +32,7 @@ struct k3_udma_glue_common { bool epib; u32 psdata_size; u32 swdata_size; + u32 atype; }; struct k3_udma_glue_tx_channel { @@ -121,6 +122,15 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, return -ENOENT; thread_id = dma_spec.args[0]; + if (dma_spec.args_count == 2) { + if (dma_spec.args[1] > 2) { + dev_err(common->dev, "Invalid channel atype: %u\n", + dma_spec.args[1]); + ret = -EINVAL; + goto out_put_spec; + } + common->atype = dma_spec.args[1]; + } if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { ret = -EINVAL; @@ -202,7 +212,8 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | - TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID; + TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; req.nav_id = tisci_rm->tisci_dev_id; req.index = tx_chn->udma_tchan_id; if (tx_chn->tx_pause_on_err) @@ -216,6 +227,7 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) req.tx_supr_tdpkt = 1; req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq); + req.tx_atype = tx_chn->common.atype; return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); } @@ -502,7 +514,8 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | - TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID; + TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; req.nav_id = tisci_rm->tisci_dev_id; req.index = rx_chn->udma_rchan_id; @@ -519,6 +532,7 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) req.flowid_cnt = rx_chn->flow_num; } req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; + req.rx_atype = rx_chn->common.atype; ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); if (ret) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 0536866a58ce..a9c0251adf1a 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -149,6 +149,7 @@ struct udma_dev { struct udma_chan *channels; u32 psil_base; + u32 atype; }; struct udma_desc { @@ -192,6 +193,7 @@ struct udma_chan_config { u32 hdesc_size; /* Size of a packet descriptor in packet mode */ bool notdpkt; /* Suppress sending TDC packet */ int remote_thread_id; + u32 atype; u32 src_thread; u32 dst_thread; enum psil_endpoint_type ep_type; @@ -1569,7 +1571,8 @@ static int udma_alloc_rx_resources(struct udma_chan *uc) TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ - TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID) + TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) #define TISCI_RCHAN_VALID_PARAMS ( \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ @@ -1579,7 +1582,8 @@ static int udma_alloc_rx_resources(struct udma_chan *uc) TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ - TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID) + TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \ + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) static int udma_tisci_m2m_channel_config(struct udma_chan *uc) { @@ -1601,6 +1605,7 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; req_tx.txcq_qnum = tc_ring; + req_tx.tx_atype = ud->atype; ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) { @@ -1614,6 +1619,7 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; req_rx.rxcq_qnum = tc_ring; req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; + req_rx.rx_atype = ud->atype; ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); if (ret) @@ -1649,6 +1655,7 @@ static int udma_tisci_tx_channel_config(struct udma_chan *uc) req_tx.tx_supr_tdpkt = uc->config.notdpkt; req_tx.tx_fetch_size = fetch_size >> 2; req_tx.txcq_qnum = tc_ring; + req_tx.tx_atype = uc->config.atype; ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) @@ -1685,6 +1692,7 @@ static int udma_tisci_rx_channel_config(struct udma_chan *uc) req_rx.rx_fetch_size = fetch_size >> 2; req_rx.rxcq_qnum = rx_ring; req_rx.rx_chan_type = mode; + req_rx.rx_atype = uc->config.atype; ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); if (ret) { @@ -3063,13 +3071,18 @@ static void udma_free_chan_resources(struct dma_chan *chan) static struct platform_driver udma_driver; +struct udma_filter_param { + int remote_thread_id; + u32 atype; +}; + static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) { struct udma_chan_config *ucc; struct psil_endpoint_config *ep_config; + struct udma_filter_param *filter_param; struct udma_chan *uc; struct udma_dev *ud; - u32 *args; if (chan->device->dev->driver != &udma_driver.driver) return false; @@ -3077,9 +3090,16 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) uc = to_udma_chan(chan); ucc = &uc->config; ud = uc->ud; - args = param; + filter_param = param; - ucc->remote_thread_id = args[0]; + if (filter_param->atype > 2) { + dev_err(ud->dev, "Invalid channel atype: %u\n", + filter_param->atype); + return false; + } + + ucc->remote_thread_id = filter_param->remote_thread_id; + ucc->atype = filter_param->atype; if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) ucc->dir = DMA_MEM_TO_DEV; @@ -3092,6 +3112,7 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) ucc->remote_thread_id); ucc->dir = DMA_MEM_TO_MEM; ucc->remote_thread_id = -1; + ucc->atype = 0; return false; } @@ -3130,13 +3151,20 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, { struct udma_dev *ud = ofdma->of_dma_data; dma_cap_mask_t mask = ud->ddev.cap_mask; + struct udma_filter_param filter_param; struct dma_chan *chan; - if (dma_spec->args_count != 1) + if (dma_spec->args_count != 1 && dma_spec->args_count != 2) return NULL; - chan = __dma_request_channel(&mask, udma_dma_filter_fn, - &dma_spec->args[0], ofdma->of_node); + filter_param.remote_thread_id = dma_spec->args[0]; + if (dma_spec->args_count == 2) + filter_param.atype = dma_spec->args[1]; + else + filter_param.atype = 0; + + chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, + ofdma->of_node); if (!chan) { dev_err(ud->dev, "get channel fail in %s.\n", __func__); return ERR_PTR(-EINVAL); @@ -3473,6 +3501,66 @@ static int udma_setup_rx_flush(struct udma_dev *ud) return 0; } +#ifdef CONFIG_DEBUG_FS +static void udma_dbg_summary_show_chan(struct seq_file *s, + struct dma_chan *chan) +{ + struct udma_chan *uc = to_udma_chan(chan); + struct udma_chan_config *ucc = &uc->config; + + seq_printf(s, " %-13s| %s", dma_chan_name(chan), + chan->dbg_client_name ?: "in-use"); + seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir)); + + switch (uc->config.dir) { + case DMA_MEM_TO_MEM: + seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, + ucc->src_thread, ucc->dst_thread); + break; + case DMA_DEV_TO_MEM: + seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, + ucc->src_thread, ucc->dst_thread); + break; + case DMA_MEM_TO_DEV: + seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, + ucc->src_thread, ucc->dst_thread); + break; + default: + seq_printf(s, ")\n"); + return; + } + + if (ucc->ep_type == PSIL_EP_NATIVE) { + seq_printf(s, "PSI-L Native"); + if (ucc->metadata_size) { + seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : ""); + if (ucc->psd_size) + seq_printf(s, " PSDsize:%u", ucc->psd_size); + seq_printf(s, " ]"); + } + } else { + seq_printf(s, "PDMA"); + if (ucc->enable_acc32 || ucc->enable_burst) + seq_printf(s, "[%s%s ]", + ucc->enable_acc32 ? " ACC32" : "", + ucc->enable_burst ? " BURST" : ""); + } + + seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode"); +} + +static void udma_dbg_summary_show(struct seq_file *s, + struct dma_device *dma_dev) +{ + struct dma_chan *chan; + + list_for_each_entry(chan, &dma_dev->channels, device_node) { + if (chan->client_count) + udma_dbg_summary_show_chan(s, chan); + } +} +#endif /* CONFIG_DEBUG_FS */ + #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ @@ -3519,6 +3607,12 @@ static int udma_probe(struct platform_device *pdev) return ret; } + ret = of_property_read_u32(navss_node, "ti,udma-atype", &ud->atype); + if (!ret && ud->atype > 2) { + dev_err(dev, "Invalid atype: %u\n", ud->atype); + return -EINVAL; + } + ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; @@ -3553,6 +3647,9 @@ static int udma_probe(struct platform_device *pdev) ud->ddev.device_resume = udma_resume; ud->ddev.device_terminate_all = udma_terminate_all; ud->ddev.device_synchronize = udma_synchronize; +#ifdef CONFIG_DEBUG_FS + ud->ddev.dbg_summary_show = udma_dbg_summary_show; +#endif ud->ddev.device_free_chan_resources = udma_free_chan_resources; ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; diff --git a/drivers/dma/ti/omap-dma.c b/drivers/dma/ti/omap-dma.c index a014ab96e673..918301e17552 100644 --- a/drivers/dma/ti/omap-dma.c +++ b/drivers/dma/ti/omap-dma.c @@ -124,7 +124,7 @@ struct omap_desc { uint32_t csdp; /* CSDP value */ unsigned sglen; - struct omap_sg sg[0]; + struct omap_sg sg[]; }; enum { diff --git a/drivers/dma/uniphier-mdmac.c b/drivers/dma/uniphier-mdmac.c index 21b8f1131d55..618839df0748 100644 --- a/drivers/dma/uniphier-mdmac.c +++ b/drivers/dma/uniphier-mdmac.c @@ -68,7 +68,7 @@ struct uniphier_mdmac_device { struct dma_device ddev; struct clk *clk; void __iomem *reg_base; - struct uniphier_mdmac_chan channels[0]; + struct uniphier_mdmac_chan channels[]; }; static struct uniphier_mdmac_chan * diff --git a/drivers/dma/uniphier-xdmac.c b/drivers/dma/uniphier-xdmac.c new file mode 100644 index 000000000000..7b2f8a8c2d31 --- /dev/null +++ b/drivers/dma/uniphier-xdmac.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * External DMA controller driver for UniPhier SoCs + * Copyright 2019 Socionext Inc. + * Author: Kunihiko Hayashi + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "dmaengine.h" +#include "virt-dma.h" + +#define XDMAC_CH_WIDTH 0x100 + +#define XDMAC_TFA 0x08 +#define XDMAC_TFA_MCNT_MASK GENMASK(23, 16) +#define XDMAC_TFA_MASK GENMASK(5, 0) +#define XDMAC_SADM 0x10 +#define XDMAC_SADM_STW_MASK GENMASK(25, 24) +#define XDMAC_SADM_SAM BIT(4) +#define XDMAC_SADM_SAM_FIXED XDMAC_SADM_SAM +#define XDMAC_SADM_SAM_INC 0 +#define XDMAC_DADM 0x14 +#define XDMAC_DADM_DTW_MASK XDMAC_SADM_STW_MASK +#define XDMAC_DADM_DAM XDMAC_SADM_SAM +#define XDMAC_DADM_DAM_FIXED XDMAC_SADM_SAM_FIXED +#define XDMAC_DADM_DAM_INC XDMAC_SADM_SAM_INC +#define XDMAC_EXSAD 0x18 +#define XDMAC_EXDAD 0x1c +#define XDMAC_SAD 0x20 +#define XDMAC_DAD 0x24 +#define XDMAC_ITS 0x28 +#define XDMAC_ITS_MASK GENMASK(25, 0) +#define XDMAC_TNUM 0x2c +#define XDMAC_TNUM_MASK GENMASK(15, 0) +#define XDMAC_TSS 0x30 +#define XDMAC_TSS_REQ BIT(0) +#define XDMAC_IEN 0x34 +#define XDMAC_IEN_ERRIEN BIT(1) +#define XDMAC_IEN_ENDIEN BIT(0) +#define XDMAC_STAT 0x40 +#define XDMAC_STAT_TENF BIT(0) +#define XDMAC_IR 0x44 +#define XDMAC_IR_ERRF BIT(1) +#define XDMAC_IR_ENDF BIT(0) +#define XDMAC_ID 0x48 +#define XDMAC_ID_ERRIDF BIT(1) +#define XDMAC_ID_ENDIDF BIT(0) + +#define XDMAC_MAX_CHANS 16 +#define XDMAC_INTERVAL_CLKS 20 +#define XDMAC_MAX_WORDS XDMAC_TNUM_MASK + +/* cut lower bit for maintain alignment of maximum transfer size */ +#define XDMAC_MAX_WORD_SIZE (XDMAC_ITS_MASK & ~GENMASK(3, 0)) + +#define UNIPHIER_XDMAC_BUSWIDTHS \ + (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +struct uniphier_xdmac_desc_node { + dma_addr_t src; + dma_addr_t dst; + u32 burst_size; + u32 nr_burst; +}; + +struct uniphier_xdmac_desc { + struct virt_dma_desc vd; + + unsigned int nr_node; + unsigned int cur_node; + enum dma_transfer_direction dir; + struct uniphier_xdmac_desc_node nodes[]; +}; + +struct uniphier_xdmac_chan { + struct virt_dma_chan vc; + struct uniphier_xdmac_device *xdev; + struct uniphier_xdmac_desc *xd; + void __iomem *reg_ch_base; + struct dma_slave_config sconfig; + int id; + unsigned int req_factor; +}; + +struct uniphier_xdmac_device { + struct dma_device ddev; + void __iomem *reg_base; + int nr_chans; + struct uniphier_xdmac_chan channels[]; +}; + +static struct uniphier_xdmac_chan * +to_uniphier_xdmac_chan(struct virt_dma_chan *vc) +{ + return container_of(vc, struct uniphier_xdmac_chan, vc); +} + +static struct uniphier_xdmac_desc * +to_uniphier_xdmac_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct uniphier_xdmac_desc, vd); +} + +/* xc->vc.lock must be held by caller */ +static struct uniphier_xdmac_desc * +uniphier_xdmac_next_desc(struct uniphier_xdmac_chan *xc) +{ + struct virt_dma_desc *vd; + + vd = vchan_next_desc(&xc->vc); + if (!vd) + return NULL; + + list_del(&vd->node); + + return to_uniphier_xdmac_desc(vd); +} + +/* xc->vc.lock must be held by caller */ +static void uniphier_xdmac_chan_start(struct uniphier_xdmac_chan *xc, + struct uniphier_xdmac_desc *xd) +{ + u32 src_mode, src_addr, src_width; + u32 dst_mode, dst_addr, dst_width; + u32 val, its, tnum; + enum dma_slave_buswidth buswidth; + + src_addr = xd->nodes[xd->cur_node].src; + dst_addr = xd->nodes[xd->cur_node].dst; + its = xd->nodes[xd->cur_node].burst_size; + tnum = xd->nodes[xd->cur_node].nr_burst; + + /* + * The width of MEM side must be 4 or 8 bytes, that does not + * affect that of DEV side and transfer size. + */ + if (xd->dir == DMA_DEV_TO_MEM) { + src_mode = XDMAC_SADM_SAM_FIXED; + buswidth = xc->sconfig.src_addr_width; + } else { + src_mode = XDMAC_SADM_SAM_INC; + buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES; + } + src_width = FIELD_PREP(XDMAC_SADM_STW_MASK, __ffs(buswidth)); + + if (xd->dir == DMA_MEM_TO_DEV) { + dst_mode = XDMAC_DADM_DAM_FIXED; + buswidth = xc->sconfig.dst_addr_width; + } else { + dst_mode = XDMAC_DADM_DAM_INC; + buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES; + } + dst_width = FIELD_PREP(XDMAC_DADM_DTW_MASK, __ffs(buswidth)); + + /* setup transfer factor */ + val = FIELD_PREP(XDMAC_TFA_MCNT_MASK, XDMAC_INTERVAL_CLKS); + val |= FIELD_PREP(XDMAC_TFA_MASK, xc->req_factor); + writel(val, xc->reg_ch_base + XDMAC_TFA); + + /* setup the channel */ + writel(lower_32_bits(src_addr), xc->reg_ch_base + XDMAC_SAD); + writel(upper_32_bits(src_addr), xc->reg_ch_base + XDMAC_EXSAD); + + writel(lower_32_bits(dst_addr), xc->reg_ch_base + XDMAC_DAD); + writel(upper_32_bits(dst_addr), xc->reg_ch_base + XDMAC_EXDAD); + + src_mode |= src_width; + dst_mode |= dst_width; + writel(src_mode, xc->reg_ch_base + XDMAC_SADM); + writel(dst_mode, xc->reg_ch_base + XDMAC_DADM); + + writel(its, xc->reg_ch_base + XDMAC_ITS); + writel(tnum, xc->reg_ch_base + XDMAC_TNUM); + + /* enable interrupt */ + writel(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN, + xc->reg_ch_base + XDMAC_IEN); + + /* start XDMAC */ + val = readl(xc->reg_ch_base + XDMAC_TSS); + val |= XDMAC_TSS_REQ; + writel(val, xc->reg_ch_base + XDMAC_TSS); +} + +/* xc->vc.lock must be held by caller */ +static int uniphier_xdmac_chan_stop(struct uniphier_xdmac_chan *xc) +{ + u32 val; + + /* disable interrupt */ + val = readl(xc->reg_ch_base + XDMAC_IEN); + val &= ~(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN); + writel(val, xc->reg_ch_base + XDMAC_IEN); + + /* stop XDMAC */ + val = readl(xc->reg_ch_base + XDMAC_TSS); + val &= ~XDMAC_TSS_REQ; + writel(0, xc->reg_ch_base + XDMAC_TSS); + + /* wait until transfer is stopped */ + return readl_poll_timeout(xc->reg_ch_base + XDMAC_STAT, val, + !(val & XDMAC_STAT_TENF), 100, 1000); +} + +/* xc->vc.lock must be held by caller */ +static void uniphier_xdmac_start(struct uniphier_xdmac_chan *xc) +{ + struct uniphier_xdmac_desc *xd; + + xd = uniphier_xdmac_next_desc(xc); + if (xd) + uniphier_xdmac_chan_start(xc, xd); + + /* set desc to chan regardless of xd is null */ + xc->xd = xd; +} + +static void uniphier_xdmac_chan_irq(struct uniphier_xdmac_chan *xc) +{ + u32 stat; + int ret; + + spin_lock(&xc->vc.lock); + + stat = readl(xc->reg_ch_base + XDMAC_ID); + + if (stat & XDMAC_ID_ERRIDF) { + ret = uniphier_xdmac_chan_stop(xc); + if (ret) + dev_err(xc->xdev->ddev.dev, + "DMA transfer error with aborting issue\n"); + else + dev_err(xc->xdev->ddev.dev, + "DMA transfer error\n"); + + } else if ((stat & XDMAC_ID_ENDIDF) && xc->xd) { + xc->xd->cur_node++; + if (xc->xd->cur_node >= xc->xd->nr_node) { + vchan_cookie_complete(&xc->xd->vd); + uniphier_xdmac_start(xc); + } else { + uniphier_xdmac_chan_start(xc, xc->xd); + } + } + + /* write bits to clear */ + writel(stat, xc->reg_ch_base + XDMAC_IR); + + spin_unlock(&xc->vc.lock); +} + +static irqreturn_t uniphier_xdmac_irq_handler(int irq, void *dev_id) +{ + struct uniphier_xdmac_device *xdev = dev_id; + int i; + + for (i = 0; i < xdev->nr_chans; i++) + uniphier_xdmac_chan_irq(&xdev->channels[i]); + + return IRQ_HANDLED; +} + +static void uniphier_xdmac_free_chan_resources(struct dma_chan *chan) +{ + vchan_free_chan_resources(to_virt_chan(chan)); +} + +static struct dma_async_tx_descriptor * +uniphier_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_desc *xd; + unsigned int nr; + size_t burst_size, tlen; + int i; + + if (len > XDMAC_MAX_WORD_SIZE * XDMAC_MAX_WORDS) + return NULL; + + nr = 1 + len / XDMAC_MAX_WORD_SIZE; + + xd = kzalloc(struct_size(xd, nodes, nr), GFP_NOWAIT); + if (!xd) + return NULL; + + for (i = 0; i < nr; i++) { + burst_size = min_t(size_t, len, XDMAC_MAX_WORD_SIZE); + xd->nodes[i].src = src; + xd->nodes[i].dst = dst; + xd->nodes[i].burst_size = burst_size; + xd->nodes[i].nr_burst = len / burst_size; + tlen = rounddown(len, burst_size); + src += tlen; + dst += tlen; + len -= tlen; + } + + xd->dir = DMA_MEM_TO_MEM; + xd->nr_node = nr; + xd->cur_node = 0; + + return vchan_tx_prep(vc, &xd->vd, flags); +} + +static struct dma_async_tx_descriptor * +uniphier_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); + struct uniphier_xdmac_desc *xd; + struct scatterlist *sg; + enum dma_slave_buswidth buswidth; + u32 maxburst; + int i; + + if (!is_slave_direction(direction)) + return NULL; + + if (direction == DMA_DEV_TO_MEM) { + buswidth = xc->sconfig.src_addr_width; + maxburst = xc->sconfig.src_maxburst; + } else { + buswidth = xc->sconfig.dst_addr_width; + maxburst = xc->sconfig.dst_maxburst; + } + + if (!maxburst) + maxburst = 1; + if (maxburst > xc->xdev->ddev.max_burst) { + dev_err(xc->xdev->ddev.dev, + "Exceed maximum number of burst words\n"); + return NULL; + } + + xd = kzalloc(struct_size(xd, nodes, sg_len), GFP_NOWAIT); + if (!xd) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) { + xd->nodes[i].src = (direction == DMA_DEV_TO_MEM) + ? xc->sconfig.src_addr : sg_dma_address(sg); + xd->nodes[i].dst = (direction == DMA_MEM_TO_DEV) + ? xc->sconfig.dst_addr : sg_dma_address(sg); + xd->nodes[i].burst_size = maxburst * buswidth; + xd->nodes[i].nr_burst = + sg_dma_len(sg) / xd->nodes[i].burst_size; + + /* + * Currently transfer that size doesn't align the unit size + * (the number of burst words * bus-width) is not allowed, + * because the driver does not support the way to transfer + * residue size. As a matter of fact, in order to transfer + * arbitrary size, 'src_maxburst' or 'dst_maxburst' of + * dma_slave_config must be 1. + */ + if (sg_dma_len(sg) % xd->nodes[i].burst_size) { + dev_err(xc->xdev->ddev.dev, + "Unaligned transfer size: %d", sg_dma_len(sg)); + kfree(xd); + return NULL; + } + + if (xd->nodes[i].nr_burst > XDMAC_MAX_WORDS) { + dev_err(xc->xdev->ddev.dev, + "Exceed maximum transfer size"); + kfree(xd); + return NULL; + } + } + + xd->dir = direction; + xd->nr_node = sg_len; + xd->cur_node = 0; + + return vchan_tx_prep(vc, &xd->vd, flags); +} + +static int uniphier_xdmac_slave_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); + + memcpy(&xc->sconfig, config, sizeof(*config)); + + return 0; +} + +static int uniphier_xdmac_terminate_all(struct dma_chan *chan) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); + unsigned long flags; + int ret = 0; + LIST_HEAD(head); + + spin_lock_irqsave(&vc->lock, flags); + + if (xc->xd) { + vchan_terminate_vdesc(&xc->xd->vd); + xc->xd = NULL; + ret = uniphier_xdmac_chan_stop(xc); + } + + vchan_get_all_descriptors(vc, &head); + + spin_unlock_irqrestore(&vc->lock, flags); + + vchan_dma_desc_free_list(vc, &head); + + return ret; +} + +static void uniphier_xdmac_synchronize(struct dma_chan *chan) +{ + vchan_synchronize(to_virt_chan(chan)); +} + +static void uniphier_xdmac_issue_pending(struct dma_chan *chan) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); + unsigned long flags; + + spin_lock_irqsave(&vc->lock, flags); + + if (vchan_issue_pending(vc) && !xc->xd) + uniphier_xdmac_start(xc); + + spin_unlock_irqrestore(&vc->lock, flags); +} + +static void uniphier_xdmac_desc_free(struct virt_dma_desc *vd) +{ + kfree(to_uniphier_xdmac_desc(vd)); +} + +static void uniphier_xdmac_chan_init(struct uniphier_xdmac_device *xdev, + int ch) +{ + struct uniphier_xdmac_chan *xc = &xdev->channels[ch]; + + xc->xdev = xdev; + xc->reg_ch_base = xdev->reg_base + XDMAC_CH_WIDTH * ch; + xc->vc.desc_free = uniphier_xdmac_desc_free; + + vchan_init(&xc->vc, &xdev->ddev); +} + +static struct dma_chan *of_dma_uniphier_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct uniphier_xdmac_device *xdev = ofdma->of_dma_data; + int chan_id = dma_spec->args[0]; + + if (chan_id >= xdev->nr_chans) + return NULL; + + xdev->channels[chan_id].id = chan_id; + xdev->channels[chan_id].req_factor = dma_spec->args[1]; + + return dma_get_slave_channel(&xdev->channels[chan_id].vc.chan); +} + +static int uniphier_xdmac_probe(struct platform_device *pdev) +{ + struct uniphier_xdmac_device *xdev; + struct device *dev = &pdev->dev; + struct dma_device *ddev; + int irq; + int nr_chans; + int i, ret; + + if (of_property_read_u32(dev->of_node, "dma-channels", &nr_chans)) + return -EINVAL; + if (nr_chans > XDMAC_MAX_CHANS) + nr_chans = XDMAC_MAX_CHANS; + + xdev = devm_kzalloc(dev, struct_size(xdev, channels, nr_chans), + GFP_KERNEL); + if (!xdev) + return -ENOMEM; + + xdev->nr_chans = nr_chans; + xdev->reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(xdev->reg_base)) + return PTR_ERR(xdev->reg_base); + + ddev = &xdev->ddev; + ddev->dev = dev; + dma_cap_zero(ddev->cap_mask); + dma_cap_set(DMA_MEMCPY, ddev->cap_mask); + dma_cap_set(DMA_SLAVE, ddev->cap_mask); + ddev->src_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS; + ddev->dst_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS; + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | + BIT(DMA_MEM_TO_MEM); + ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + ddev->max_burst = XDMAC_MAX_WORDS; + ddev->device_free_chan_resources = uniphier_xdmac_free_chan_resources; + ddev->device_prep_dma_memcpy = uniphier_xdmac_prep_dma_memcpy; + ddev->device_prep_slave_sg = uniphier_xdmac_prep_slave_sg; + ddev->device_config = uniphier_xdmac_slave_config; + ddev->device_terminate_all = uniphier_xdmac_terminate_all; + ddev->device_synchronize = uniphier_xdmac_synchronize; + ddev->device_tx_status = dma_cookie_status; + ddev->device_issue_pending = uniphier_xdmac_issue_pending; + INIT_LIST_HEAD(&ddev->channels); + + for (i = 0; i < nr_chans; i++) + uniphier_xdmac_chan_init(xdev, i); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(dev, irq, uniphier_xdmac_irq_handler, + IRQF_SHARED, "xdmac", xdev); + if (ret) { + dev_err(dev, "Failed to request IRQ\n"); + return ret; + } + + ret = dma_async_device_register(ddev); + if (ret) { + dev_err(dev, "Failed to register XDMA device\n"); + return ret; + } + + ret = of_dma_controller_register(dev->of_node, + of_dma_uniphier_xlate, xdev); + if (ret) { + dev_err(dev, "Failed to register XDMA controller\n"); + goto out_unregister_dmac; + } + + platform_set_drvdata(pdev, xdev); + + dev_info(&pdev->dev, "UniPhier XDMAC driver (%d channels)\n", + nr_chans); + + return 0; + +out_unregister_dmac: + dma_async_device_unregister(ddev); + + return ret; +} + +static int uniphier_xdmac_remove(struct platform_device *pdev) +{ + struct uniphier_xdmac_device *xdev = platform_get_drvdata(pdev); + struct dma_device *ddev = &xdev->ddev; + struct dma_chan *chan; + int ret; + + /* + * Before reaching here, almost all descriptors have been freed by the + * ->device_free_chan_resources() hook. However, each channel might + * be still holding one descriptor that was on-flight at that moment. + * Terminate it to make sure this hardware is no longer running. Then, + * free the channel resources once again to avoid memory leak. + */ + list_for_each_entry(chan, &ddev->channels, device_node) { + ret = dmaengine_terminate_sync(chan); + if (ret) + return ret; + uniphier_xdmac_free_chan_resources(chan); + } + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(ddev); + + return 0; +} + +static const struct of_device_id uniphier_xdmac_match[] = { + { .compatible = "socionext,uniphier-xdmac" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, uniphier_xdmac_match); + +static struct platform_driver uniphier_xdmac_driver = { + .probe = uniphier_xdmac_probe, + .remove = uniphier_xdmac_remove, + .driver = { + .name = "uniphier-xdmac", + .of_match_table = uniphier_xdmac_match, + }, +}; +module_platform_driver(uniphier_xdmac_driver); + +MODULE_AUTHOR("Kunihiko Hayashi "); +MODULE_DESCRIPTION("UniPhier external DMA controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index a9c5d5cc9f2b..aecd5a35a296 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -125,7 +125,9 @@ #define XILINX_VDMA_ENABLE_VERTICAL_FLIP BIT(0) /* HW specific definitions */ -#define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x20 +#define XILINX_MCDMA_MAX_CHANS_PER_DEVICE 0x20 +#define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x2 +#define XILINX_CDMA_MAX_CHANS_PER_DEVICE 0x1 #define XILINX_DMA_DMAXR_ALL_IRQ_MASK \ (XILINX_DMA_DMASR_FRM_CNT_IRQ | \ @@ -468,6 +470,7 @@ struct xilinx_dma_config { struct clk **tx_clk, struct clk **txs_clk, struct clk **rx_clk, struct clk **rxs_clk); irqreturn_t (*irq_handler)(int irq, void *data); + const int max_channels; }; /** @@ -485,16 +488,15 @@ struct xilinx_dma_config { * @txs_clk: DMA mm2s stream clock * @rx_clk: DMA s2mm clock * @rxs_clk: DMA s2mm stream clock - * @nr_channels: Number of channels DMA device supports - * @chan_id: DMA channel identifier + * @s2mm_chan_id: DMA s2mm channel identifier + * @mm2s_chan_id: DMA mm2s channel identifier * @max_buffer_len: Max buffer length - * @s2mm_index: S2MM channel index */ struct xilinx_dma_device { void __iomem *regs; struct device *dev; struct dma_device common; - struct xilinx_dma_chan *chan[XILINX_DMA_MAX_CHANS_PER_DEVICE]; + struct xilinx_dma_chan *chan[XILINX_MCDMA_MAX_CHANS_PER_DEVICE]; u32 flush_on_fsync; bool ext_addr; struct platform_device *pdev; @@ -504,10 +506,9 @@ struct xilinx_dma_device { struct clk *txs_clk; struct clk *rx_clk; struct clk *rxs_clk; - u32 nr_channels; - u32 chan_id; + u32 s2mm_chan_id; + u32 mm2s_chan_id; u32 max_buffer_len; - u32 s2mm_index; }; /* Macros */ @@ -1745,7 +1746,7 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) return IRQ_NONE; if (chan->direction == DMA_DEV_TO_MEM) - chan_offset = chan->xdev->s2mm_index; + chan_offset = chan->xdev->dma_config->max_channels / 2; chan_offset = chan_offset + (chan_id - 1); chan = chan->xdev->chan[chan_offset]; @@ -2404,16 +2405,17 @@ static int xilinx_dma_terminate_all(struct dma_chan *dchan) u32 reg; int err; - if (chan->cyclic) - xilinx_dma_chan_reset(chan); - - err = chan->stop_transfer(chan); - if (err) { - dev_err(chan->dev, "Cannot stop channel %p: %x\n", - chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); - chan->err = true; + if (!chan->cyclic) { + err = chan->stop_transfer(chan); + if (err) { + dev_err(chan->dev, "Cannot stop channel %p: %x\n", + chan, dma_ctrl_read(chan, + XILINX_DMA_REG_DMASR)); + chan->err = true; + } } + xilinx_dma_chan_reset(chan); /* Remove and free all of the descriptors in the lists */ xilinx_dma_free_descriptors(chan); chan->idle = true; @@ -2730,12 +2732,11 @@ static void xdma_disable_allclks(struct xilinx_dma_device *xdev) * * @xdev: Driver specific device structure * @node: Device node - * @chan_id: DMA Channel id * * Return: '0' on success and failure value on error */ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, - struct device_node *node, int chan_id) + struct device_node *node) { struct xilinx_dma_chan *chan; bool has_dre = false; @@ -2787,8 +2788,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") || of_device_is_compatible(node, "xlnx,axi-cdma-channel")) { chan->direction = DMA_MEM_TO_DEV; - chan->id = chan_id; - chan->tdest = chan_id; + chan->id = xdev->mm2s_chan_id++; + chan->tdest = chan->id; chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET; if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { @@ -2804,9 +2805,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, of_device_is_compatible(node, "xlnx,axi-dma-s2mm-channel")) { chan->direction = DMA_DEV_TO_MEM; - chan->id = chan_id; - xdev->s2mm_index = xdev->nr_channels; - chan->tdest = chan_id - xdev->nr_channels; + chan->id = xdev->s2mm_chan_id++; + chan->tdest = chan->id - xdev->dma_config->max_channels / 2; chan->has_vflip = of_property_read_bool(node, "xlnx,enable-vert-flip"); if (chan->has_vflip) { @@ -2908,9 +2908,7 @@ static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, dev_warn(xdev->dev, "missing dma-channels property\n"); for (i = 0; i < nr_channels; i++) - xilinx_dma_chan_probe(xdev, node, xdev->chan_id++); - - xdev->nr_channels += nr_channels; + xilinx_dma_chan_probe(xdev, node); return 0; } @@ -2928,7 +2926,7 @@ static struct dma_chan *of_dma_xilinx_xlate(struct of_phandle_args *dma_spec, struct xilinx_dma_device *xdev = ofdma->of_dma_data; int chan_id = dma_spec->args[0]; - if (chan_id >= xdev->nr_channels || !xdev->chan[chan_id]) + if (chan_id >= xdev->dma_config->max_channels || !xdev->chan[chan_id]) return NULL; return dma_get_slave_channel(&xdev->chan[chan_id]->common); @@ -2938,23 +2936,27 @@ static const struct xilinx_dma_config axidma_config = { .dmatype = XDMA_TYPE_AXIDMA, .clk_init = axidma_clk_init, .irq_handler = xilinx_dma_irq_handler, + .max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE, }; static const struct xilinx_dma_config aximcdma_config = { .dmatype = XDMA_TYPE_AXIMCDMA, .clk_init = axidma_clk_init, .irq_handler = xilinx_mcdma_irq_handler, + .max_channels = XILINX_MCDMA_MAX_CHANS_PER_DEVICE, }; static const struct xilinx_dma_config axicdma_config = { .dmatype = XDMA_TYPE_CDMA, .clk_init = axicdma_clk_init, .irq_handler = xilinx_dma_irq_handler, + .max_channels = XILINX_CDMA_MAX_CHANS_PER_DEVICE, }; static const struct xilinx_dma_config axivdma_config = { .dmatype = XDMA_TYPE_VDMA, .clk_init = axivdma_clk_init, .irq_handler = xilinx_dma_irq_handler, + .max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE, }; static const struct of_device_id xilinx_dma_of_ids[] = { @@ -3011,6 +3013,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) /* Retrieve the DMA engine properties from the device tree */ xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0); + xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2; if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA || xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { @@ -3104,7 +3107,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) } if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { - for (i = 0; i < xdev->nr_channels; i++) + for (i = 0; i < xdev->dma_config->max_channels; i++) if (xdev->chan[i]) xdev->chan[i]->num_frms = num_frames; } @@ -3134,7 +3137,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) disable_clks: xdma_disable_allclks(xdev); error: - for (i = 0; i < xdev->nr_channels; i++) + for (i = 0; i < xdev->dma_config->max_channels; i++) if (xdev->chan[i]) xilinx_dma_chan_remove(xdev->chan[i]); @@ -3156,7 +3159,7 @@ static int xilinx_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&xdev->common); - for (i = 0; i < xdev->nr_channels; i++) + for (i = 0; i < xdev->dma_config->max_channels; i++) if (xdev->chan[i]) xilinx_dma_chan_remove(xdev->chan[i]); diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 00fb91feba70..86bde1e30a60 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -149,6 +149,7 @@ struct gpio_desc *gpiochip_get_desc(struct gpio_chip *chip, return &gdev->descs[hwnum]; } +EXPORT_SYMBOL_GPL(gpiochip_get_desc); /** * desc_to_gpio - convert a GPIO descriptor to the integer namespace diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c index a8b20557539b..ff43a3d80410 100644 --- a/drivers/gpu/drm/mediatek/mtk_hdmi.c +++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -169,6 +170,9 @@ struct mtk_hdmi { bool audio_enable; bool powered; bool enabled; + hdmi_codec_plugged_cb plugged_cb; + struct device *codec_dev; + struct mutex update_plugged_status_lock; }; static inline struct mtk_hdmi *hdmi_ctx_from_bridge(struct drm_bridge *b) @@ -1194,13 +1198,26 @@ static void mtk_hdmi_clk_disable_audio(struct mtk_hdmi *hdmi) clk_disable_unprepare(hdmi->clk[MTK_HDMI_CLK_AUD_SPDIF]); } +static enum drm_connector_status +mtk_hdmi_update_plugged_status(struct mtk_hdmi *hdmi) +{ + bool connected; + + mutex_lock(&hdmi->update_plugged_status_lock); + connected = mtk_cec_hpd_high(hdmi->cec_dev); + if (hdmi->plugged_cb && hdmi->codec_dev) + hdmi->plugged_cb(hdmi->codec_dev, connected); + mutex_unlock(&hdmi->update_plugged_status_lock); + + return connected ? + connector_status_connected : connector_status_disconnected; +} + static enum drm_connector_status hdmi_conn_detect(struct drm_connector *conn, bool force) { struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn); - - return mtk_cec_hpd_high(hdmi->cec_dev) ? - connector_status_connected : connector_status_disconnected; + return mtk_hdmi_update_plugged_status(hdmi); } static void hdmi_conn_destroy(struct drm_connector *conn) @@ -1657,20 +1674,39 @@ static int mtk_hdmi_audio_get_eld(struct device *dev, void *data, uint8_t *buf, return 0; } +static int mtk_hdmi_audio_hook_plugged_cb(struct device *dev, void *data, + hdmi_codec_plugged_cb fn, + struct device *codec_dev) +{ + struct mtk_hdmi *hdmi = data; + + mutex_lock(&hdmi->update_plugged_status_lock); + hdmi->plugged_cb = fn; + hdmi->codec_dev = codec_dev; + mutex_unlock(&hdmi->update_plugged_status_lock); + + mtk_hdmi_update_plugged_status(hdmi); + + return 0; +} + static const struct hdmi_codec_ops mtk_hdmi_audio_codec_ops = { .hw_params = mtk_hdmi_audio_hw_params, .audio_startup = mtk_hdmi_audio_startup, .audio_shutdown = mtk_hdmi_audio_shutdown, .digital_mute = mtk_hdmi_audio_digital_mute, .get_eld = mtk_hdmi_audio_get_eld, + .hook_plugged_cb = mtk_hdmi_audio_hook_plugged_cb, }; -static void mtk_hdmi_register_audio_driver(struct device *dev) +static int mtk_hdmi_register_audio_driver(struct device *dev) { + struct mtk_hdmi *hdmi = dev_get_drvdata(dev); struct hdmi_codec_pdata codec_data = { .ops = &mtk_hdmi_audio_codec_ops, .max_i2s_channels = 2, .i2s = 1, + .data = hdmi, }; struct platform_device *pdev; @@ -1678,9 +1714,10 @@ static void mtk_hdmi_register_audio_driver(struct device *dev) PLATFORM_DEVID_AUTO, &codec_data, sizeof(codec_data)); if (IS_ERR(pdev)) - return; + return PTR_ERR(pdev); DRM_INFO("%s driver bound to HDMI\n", HDMI_CODEC_DRV_NAME); + return 0; } static int mtk_drm_hdmi_probe(struct platform_device *pdev) @@ -1706,6 +1743,7 @@ static int mtk_drm_hdmi_probe(struct platform_device *pdev) return ret; } + mutex_init(&hdmi->update_plugged_status_lock); platform_set_drvdata(pdev, hdmi); ret = mtk_hdmi_output_init(hdmi); @@ -1714,7 +1752,11 @@ static int mtk_drm_hdmi_probe(struct platform_device *pdev) return ret; } - mtk_hdmi_register_audio_driver(dev); + ret = mtk_hdmi_register_audio_driver(dev); + if (ret) { + dev_err(dev, "Failed to register audio driver: %d\n", ret); + return ret; + } hdmi->bridge.funcs = &mtk_hdmi_bridge_funcs; hdmi->bridge.of_node = pdev->dev.of_node; diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 389128b8c4dd..cb8829ca6c7f 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -59,9 +59,10 @@ static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, /* * If possible, avoid waiting for GPU with mmap_sem - * held. + * held. We only do this if the fault allows retry and this + * is the first attempt. */ - if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + if (fault_flag_allow_retry_first(vmf->flags)) { ret = VM_FAULT_RETRY; if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) goto out_unlock; @@ -135,7 +136,12 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, * for the buffer to become unreserved. */ if (unlikely(!dma_resv_trylock(bo->base.resv))) { - if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + /* + * If the fault allows retry and this is the first + * fault attempt, we try to release the mmap_sem + * before waiting + */ + if (fault_flag_allow_retry_first(vmf->flags)) { if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { ttm_bo_get(bo); up_read(&vmf->vma->vm_mm->mmap_sem); diff --git a/drivers/hwspinlock/Kconfig b/drivers/hwspinlock/Kconfig index 37740e992cfa..826a1054100d 100644 --- a/drivers/hwspinlock/Kconfig +++ b/drivers/hwspinlock/Kconfig @@ -9,7 +9,7 @@ menuconfig HWSPINLOCK config HWSPINLOCK_OMAP tristate "OMAP Hardware Spinlock device" depends on HWSPINLOCK - depends on ARCH_OMAP4 || SOC_OMAP5 || SOC_DRA7XX || SOC_AM33XX || SOC_AM43XX || ARCH_K3 + depends on ARCH_OMAP4 || SOC_OMAP5 || SOC_DRA7XX || SOC_AM33XX || SOC_AM43XX || ARCH_K3 || COMPILE_TEST help Say y here to support the OMAP Hardware Spinlock device (firstly introduced in OMAP4). @@ -19,7 +19,7 @@ config HWSPINLOCK_OMAP config HWSPINLOCK_QCOM tristate "Qualcomm Hardware Spinlock device" depends on HWSPINLOCK - depends on ARCH_QCOM + depends on ARCH_QCOM || COMPILE_TEST select MFD_SYSCON help Say y here to support the Qualcomm Hardware Mutex functionality, which @@ -31,7 +31,7 @@ config HWSPINLOCK_QCOM config HWSPINLOCK_SIRF tristate "SIRF Hardware Spinlock device" depends on HWSPINLOCK - depends on ARCH_SIRF + depends on ARCH_SIRF || COMPILE_TEST help Say y here to support the SIRF Hardware Spinlock device, which provides a synchronisation mechanism for the various processors @@ -42,7 +42,7 @@ config HWSPINLOCK_SIRF config HWSPINLOCK_SPRD tristate "SPRD Hardware Spinlock device" - depends on ARCH_SPRD + depends on ARCH_SPRD || COMPILE_TEST depends on HWSPINLOCK help Say y here to support the SPRD Hardware Spinlock device. @@ -51,7 +51,7 @@ config HWSPINLOCK_SPRD config HWSPINLOCK_STM32 tristate "STM32 Hardware Spinlock device" - depends on MACH_STM32MP157 + depends on MACH_STM32MP157 || COMPILE_TEST depends on HWSPINLOCK help Say y here to support the STM32 Hardware Spinlock device. @@ -61,7 +61,7 @@ config HWSPINLOCK_STM32 config HSEM_U8500 tristate "STE Hardware Semaphore functionality" depends on HWSPINLOCK - depends on ARCH_U8500 + depends on ARCH_U8500 || COMPILE_TEST help Say y here to support the STE Hardware Semaphore functionality, which provides a synchronisation mechanism for the various processor on the diff --git a/drivers/hwspinlock/hwspinlock_internal.h b/drivers/hwspinlock/hwspinlock_internal.h index 9eb6bd020dc7..29892767bb7a 100644 --- a/drivers/hwspinlock/hwspinlock_internal.h +++ b/drivers/hwspinlock/hwspinlock_internal.h @@ -56,7 +56,7 @@ struct hwspinlock_device { const struct hwspinlock_ops *ops; int base_id; int num_locks; - struct hwspinlock lock[0]; + struct hwspinlock lock[]; }; static inline int hwlock_to_id(struct hwspinlock *hwlock) diff --git a/drivers/i2c/algos/i2c-algo-pca.c b/drivers/i2c/algos/i2c-algo-pca.c index 5ac93f41bfec..dff4e178c732 100644 --- a/drivers/i2c/algos/i2c-algo-pca.c +++ b/drivers/i2c/algos/i2c-algo-pca.c @@ -459,17 +459,17 @@ static int pca_init(struct i2c_adapter *adap) /* To avoid integer overflow, use clock/100 for calculations */ clock = pca_clock(pca_data) / 100; - if (pca_data->i2c_clock > 1000000) { + if (pca_data->i2c_clock > I2C_MAX_FAST_MODE_PLUS_FREQ) { mode = I2C_PCA_MODE_TURBO; min_tlow = 14; min_thi = 5; raise_fall_time = 22; /* Raise 11e-8s, Fall 11e-8s */ - } else if (pca_data->i2c_clock > 400000) { + } else if (pca_data->i2c_clock > I2C_MAX_FAST_MODE_FREQ) { mode = I2C_PCA_MODE_FASTP; min_tlow = 17; min_thi = 9; raise_fall_time = 22; /* Raise 11e-8s, Fall 11e-8s */ - } else if (pca_data->i2c_clock > 100000) { + } else if (pca_data->i2c_clock > I2C_MAX_STANDARD_MODE_FREQ) { mode = I2C_PCA_MODE_FAST; min_tlow = 44; min_thi = 20; diff --git a/drivers/i2c/busses/i2c-altera.c b/drivers/i2c/busses/i2c-altera.c index 1de23b4f3809..20ef63820c77 100644 --- a/drivers/i2c/busses/i2c-altera.c +++ b/drivers/i2c/busses/i2c-altera.c @@ -147,7 +147,7 @@ static void altr_i2c_init(struct altr_i2c_dev *idev) (ALTR_I2C_THRESHOLD << ALTR_I2C_CTRL_TCT_SHFT); u32 t_high, t_low; - if (idev->bus_clk_rate <= 100000) { + if (idev->bus_clk_rate <= I2C_MAX_STANDARD_MODE_FREQ) { tmp &= ~ALTR_I2C_CTRL_BSPEED; /* Standard mode SCL 50/50 */ t_high = divisor * 1 / 2; @@ -423,10 +423,10 @@ static int altr_i2c_probe(struct platform_device *pdev) &idev->bus_clk_rate); if (val) { dev_err(&pdev->dev, "Default to 100kHz\n"); - idev->bus_clk_rate = 100000; /* default clock rate */ + idev->bus_clk_rate = I2C_MAX_STANDARD_MODE_FREQ; /* default clock rate */ } - if (idev->bus_clk_rate > 400000) { + if (idev->bus_clk_rate > I2C_MAX_FAST_MODE_FREQ) { dev_err(&pdev->dev, "invalid clock-frequency %d\n", idev->bus_clk_rate); return -EINVAL; diff --git a/drivers/i2c/busses/i2c-amd-mp2-plat.c b/drivers/i2c/busses/i2c-amd-mp2-plat.c index f5b3f00c6559..17df9e8845b6 100644 --- a/drivers/i2c/busses/i2c-amd-mp2-plat.c +++ b/drivers/i2c/busses/i2c-amd-mp2-plat.c @@ -201,32 +201,37 @@ static int i2c_amd_resume(struct amd_i2c_common *i2c_common) } #endif +static const u32 supported_speeds[] = { + I2C_MAX_HIGH_SPEED_MODE_FREQ, + I2C_MAX_TURBO_MODE_FREQ, + I2C_MAX_FAST_MODE_PLUS_FREQ, + I2C_MAX_FAST_MODE_FREQ, + I2C_MAX_STANDARD_MODE_FREQ, +}; + static enum speed_enum i2c_amd_get_bus_speed(struct platform_device *pdev) { u32 acpi_speed; int i; - static const u32 supported_speeds[] = { - 0, 100000, 400000, 1000000, 1400000, 3400000 - }; acpi_speed = i2c_acpi_find_bus_speed(&pdev->dev); /* round down to the lowest standard speed */ - for (i = 1; i < ARRAY_SIZE(supported_speeds); i++) { - if (acpi_speed < supported_speeds[i]) + for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) { + if (acpi_speed >= supported_speeds[i]) break; } - acpi_speed = supported_speeds[i - 1]; + acpi_speed = i < ARRAY_SIZE(supported_speeds) ? supported_speeds[i] : 0; switch (acpi_speed) { - case 100000: + case I2C_MAX_STANDARD_MODE_FREQ: return speed100k; - case 400000: + case I2C_MAX_FAST_MODE_FREQ: return speed400k; - case 1000000: + case I2C_MAX_FAST_MODE_PLUS_FREQ: return speed1000k; - case 1400000: + case I2C_MAX_TURBO_MODE_FREQ: return speed1400k; - case 3400000: + case I2C_MAX_HIGH_SPEED_MODE_FREQ: return speed3400k; default: return speed400k; diff --git a/drivers/i2c/busses/i2c-aspeed.c b/drivers/i2c/busses/i2c-aspeed.c index a7be6f24450b..07c1993274c5 100644 --- a/drivers/i2c/busses/i2c-aspeed.c +++ b/drivers/i2c/busses/i2c-aspeed.c @@ -997,7 +997,7 @@ static int aspeed_i2c_probe_bus(struct platform_device *pdev) if (ret < 0) { dev_err(&pdev->dev, "Could not read bus-frequency property\n"); - bus->bus_frequency = 100000; + bus->bus_frequency = I2C_MAX_STANDARD_MODE_FREQ; } match = of_match_node(aspeed_i2c_bus_of_table, pdev->dev.of_node); diff --git a/drivers/i2c/busses/i2c-at91-master.c b/drivers/i2c/busses/i2c-at91-master.c index 7a862e00b475..0aba51a7df32 100644 --- a/drivers/i2c/busses/i2c-at91-master.c +++ b/drivers/i2c/busses/i2c-at91-master.c @@ -18,11 +18,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -478,6 +480,7 @@ static int at91_do_twi_transfer(struct at91_twi_dev *dev) unsigned long time_left; bool has_unre_flag = dev->pdata->has_unre_flag; bool has_alt_cmd = dev->pdata->has_alt_cmd; + struct i2c_bus_recovery_info *rinfo = &dev->rinfo; /* * WARNING: the TXCOMP bit in the Status Register is NOT a clear on @@ -637,6 +640,13 @@ static int at91_do_twi_transfer(struct at91_twi_dev *dev) at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_THRCLR | AT91_TWI_LOCKCLR); } + + if (rinfo->get_sda && !(rinfo->get_sda(&dev->adapter))) { + dev_dbg(dev->dev, + "SDA is down; clear bus using gpio\n"); + i2c_recover_bus(&dev->adapter); + } + return ret; } @@ -806,6 +816,70 @@ static int at91_twi_configure_dma(struct at91_twi_dev *dev, u32 phy_addr) return ret; } +static void at91_prepare_twi_recovery(struct i2c_adapter *adap) +{ + struct at91_twi_dev *dev = i2c_get_adapdata(adap); + + pinctrl_select_state(dev->pinctrl, dev->pinctrl_pins_gpio); +} + +static void at91_unprepare_twi_recovery(struct i2c_adapter *adap) +{ + struct at91_twi_dev *dev = i2c_get_adapdata(adap); + + pinctrl_select_state(dev->pinctrl, dev->pinctrl_pins_default); +} + +static int at91_init_twi_recovery_info(struct platform_device *pdev, + struct at91_twi_dev *dev) +{ + struct i2c_bus_recovery_info *rinfo = &dev->rinfo; + + dev->pinctrl = devm_pinctrl_get(&pdev->dev); + if (!dev->pinctrl || IS_ERR(dev->pinctrl)) { + dev_info(dev->dev, "can't get pinctrl, bus recovery not supported\n"); + return PTR_ERR(dev->pinctrl); + } + + dev->pinctrl_pins_default = pinctrl_lookup_state(dev->pinctrl, + PINCTRL_STATE_DEFAULT); + dev->pinctrl_pins_gpio = pinctrl_lookup_state(dev->pinctrl, + "gpio"); + rinfo->sda_gpiod = devm_gpiod_get(&pdev->dev, "sda", GPIOD_IN); + if (PTR_ERR(rinfo->sda_gpiod) == -EPROBE_DEFER) + return -EPROBE_DEFER; + + rinfo->scl_gpiod = devm_gpiod_get(&pdev->dev, "scl", + GPIOD_OUT_HIGH_OPEN_DRAIN); + if (PTR_ERR(rinfo->scl_gpiod) == -EPROBE_DEFER) + return -EPROBE_DEFER; + + if (IS_ERR(rinfo->sda_gpiod) || + IS_ERR(rinfo->scl_gpiod) || + IS_ERR(dev->pinctrl_pins_default) || + IS_ERR(dev->pinctrl_pins_gpio)) { + dev_info(&pdev->dev, "recovery information incomplete\n"); + if (!IS_ERR(rinfo->sda_gpiod)) { + gpiod_put(rinfo->sda_gpiod); + rinfo->sda_gpiod = NULL; + } + if (!IS_ERR(rinfo->scl_gpiod)) { + gpiod_put(rinfo->scl_gpiod); + rinfo->scl_gpiod = NULL; + } + return -EINVAL; + } + + dev_info(&pdev->dev, "using scl, sda for recovery\n"); + + rinfo->prepare_recovery = at91_prepare_twi_recovery; + rinfo->unprepare_recovery = at91_unprepare_twi_recovery; + rinfo->recover_bus = i2c_generic_scl_recovery; + dev->adapter.bus_recovery_info = rinfo; + + return 0; +} + int at91_twi_probe_master(struct platform_device *pdev, u32 phy_addr, struct at91_twi_dev *dev) { @@ -838,6 +912,10 @@ int at91_twi_probe_master(struct platform_device *pdev, "i2c-analog-filter"); at91_calc_twi_clock(dev); + rc = at91_init_twi_recovery_info(pdev, dev); + if (rc == -EPROBE_DEFER) + return rc; + dev->adapter.algo = &at91_twi_algorithm; dev->adapter.quirks = &at91_twi_quirks; diff --git a/drivers/i2c/busses/i2c-at91.h b/drivers/i2c/busses/i2c-at91.h index 977a67bc0f88..f57a6cab96b4 100644 --- a/drivers/i2c/busses/i2c-at91.h +++ b/drivers/i2c/busses/i2c-at91.h @@ -151,6 +151,10 @@ struct at91_twi_dev { u32 fifo_size; struct at91_twi_dma dma; bool slave_detected; + struct i2c_bus_recovery_info rinfo; + struct pinctrl *pinctrl; + struct pinctrl_state *pinctrl_pins_default; + struct pinctrl_state *pinctrl_pins_gpio; #ifdef CONFIG_I2C_AT91_SLAVE_EXPERIMENTAL unsigned smr; struct i2c_client *slave; diff --git a/drivers/i2c/busses/i2c-axxia.c b/drivers/i2c/busses/i2c-axxia.c index 0214daa913ff..be3681d08a8d 100644 --- a/drivers/i2c/busses/i2c-axxia.c +++ b/drivers/i2c/busses/i2c-axxia.c @@ -199,7 +199,7 @@ static int axxia_i2c_init(struct axxia_i2c_dev *idev) /* Enable Master Mode */ writel(0x1, idev->base + GLOBAL_CONTROL); - if (idev->bus_clk_rate <= 100000) { + if (idev->bus_clk_rate <= I2C_MAX_STANDARD_MODE_FREQ) { /* Standard mode SCL 50/50, tSU:DAT = 250 ns */ t_high = divisor * 1 / 2; t_low = divisor * 1 / 2; @@ -765,7 +765,7 @@ static int axxia_i2c_probe(struct platform_device *pdev) of_property_read_u32(np, "clock-frequency", &idev->bus_clk_rate); if (idev->bus_clk_rate == 0) - idev->bus_clk_rate = 100000; /* default clock rate */ + idev->bus_clk_rate = I2C_MAX_STANDARD_MODE_FREQ; /* default clock rate */ ret = clk_prepare_enable(idev->i2c_clk); if (ret) { diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c index 30efb7913b2e..44be0926b566 100644 --- a/drivers/i2c/busses/i2c-bcm-iproc.c +++ b/drivers/i2c/busses/i2c-bcm-iproc.c @@ -858,25 +858,25 @@ static int bcm_iproc_i2c_cfg_speed(struct bcm_iproc_i2c_dev *iproc_i2c) if (ret < 0) { dev_info(iproc_i2c->device, "unable to interpret clock-frequency DT property\n"); - bus_speed = 100000; + bus_speed = I2C_MAX_STANDARD_MODE_FREQ; } - if (bus_speed < 100000) { + if (bus_speed < I2C_MAX_STANDARD_MODE_FREQ) { dev_err(iproc_i2c->device, "%d Hz bus speed not supported\n", bus_speed); dev_err(iproc_i2c->device, "valid speeds are 100khz and 400khz\n"); return -EINVAL; - } else if (bus_speed < 400000) { - bus_speed = 100000; + } else if (bus_speed < I2C_MAX_FAST_MODE_FREQ) { + bus_speed = I2C_MAX_STANDARD_MODE_FREQ; } else { - bus_speed = 400000; + bus_speed = I2C_MAX_FAST_MODE_FREQ; } iproc_i2c->bus_speed = bus_speed; val = iproc_i2c_rd_reg(iproc_i2c, TIM_CFG_OFFSET); val &= ~BIT(TIM_CFG_MODE_400_SHIFT); - val |= (bus_speed == 400000) << TIM_CFG_MODE_400_SHIFT; + val |= (bus_speed == I2C_MAX_FAST_MODE_FREQ) << TIM_CFG_MODE_400_SHIFT; iproc_i2c_wr_reg(iproc_i2c, TIM_CFG_OFFSET, val); dev_info(iproc_i2c->device, "bus set to %u Hz\n", bus_speed); @@ -1029,7 +1029,7 @@ static int bcm_iproc_i2c_resume(struct device *dev) /* configure to the desired bus speed */ val = iproc_i2c_rd_reg(iproc_i2c, TIM_CFG_OFFSET); val &= ~BIT(TIM_CFG_MODE_400_SHIFT); - val |= (iproc_i2c->bus_speed == 400000) << TIM_CFG_MODE_400_SHIFT; + val |= (iproc_i2c->bus_speed == I2C_MAX_FAST_MODE_FREQ) << TIM_CFG_MODE_400_SHIFT; iproc_i2c_wr_reg(iproc_i2c, TIM_CFG_OFFSET, val); bcm_iproc_i2c_enable_disable(iproc_i2c, true); diff --git a/drivers/i2c/busses/i2c-bcm-kona.c b/drivers/i2c/busses/i2c-bcm-kona.c index 4e489a9d16fb..572aebbb254e 100644 --- a/drivers/i2c/busses/i2c-bcm-kona.c +++ b/drivers/i2c/busses/i2c-bcm-kona.c @@ -722,16 +722,16 @@ static int bcm_kona_i2c_assign_bus_speed(struct bcm_kona_i2c_dev *dev) } switch (bus_speed) { - case 100000: + case I2C_MAX_STANDARD_MODE_FREQ: dev->std_cfg = &std_cfg_table[BCM_SPD_100K]; break; - case 400000: + case I2C_MAX_FAST_MODE_FREQ: dev->std_cfg = &std_cfg_table[BCM_SPD_400K]; break; - case 1000000: + case I2C_MAX_FAST_MODE_PLUS_FREQ: dev->std_cfg = &std_cfg_table[BCM_SPD_1MHZ]; break; - case 3400000: + case I2C_MAX_HIGH_SPEED_MODE_FREQ: /* Send mastercode at 100k */ dev->std_cfg = &std_cfg_table[BCM_SPD_100K]; dev->hs_cfg = &hs_cfg_table[BCM_SPD_3P4MHZ]; diff --git a/drivers/i2c/busses/i2c-bcm2835.c b/drivers/i2c/busses/i2c-bcm2835.c index 5ab901ad615d..d9b86fcc3825 100644 --- a/drivers/i2c/busses/i2c-bcm2835.c +++ b/drivers/i2c/busses/i2c-bcm2835.c @@ -439,7 +439,7 @@ static int bcm2835_i2c_probe(struct platform_device *pdev) if (ret < 0) { dev_warn(&pdev->dev, "Could not read clock-frequency property\n"); - bus_clk_rate = 100000; + bus_clk_rate = I2C_MAX_STANDARD_MODE_FREQ; } ret = clk_set_rate_exclusive(i2c_dev->bus_clk, bus_clk_rate); diff --git a/drivers/i2c/busses/i2c-brcmstb.c b/drivers/i2c/busses/i2c-brcmstb.c index 506991596b68..169a2836922d 100644 --- a/drivers/i2c/busses/i2c-brcmstb.c +++ b/drivers/i2c/busses/i2c-brcmstb.c @@ -580,6 +580,31 @@ static void brcmstb_i2c_set_bsc_reg_defaults(struct brcmstb_i2c_dev *dev) brcmstb_i2c_set_bus_speed(dev); } +#define AUTOI2C_CTRL0 0x26c +#define AUTOI2C_CTRL0_RELEASE_BSC BIT(1) + +static int bcm2711_release_bsc(struct brcmstb_i2c_dev *dev) +{ + struct platform_device *pdev = to_platform_device(dev->device); + struct resource *iomem; + void __iomem *autoi2c; + + /* Map hardware registers */ + iomem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "auto-i2c"); + autoi2c = devm_ioremap_resource(&pdev->dev, iomem); + if (IS_ERR(autoi2c)) + return PTR_ERR(autoi2c); + + writel(AUTOI2C_CTRL0_RELEASE_BSC, autoi2c + AUTOI2C_CTRL0); + devm_iounmap(&pdev->dev, autoi2c); + + /* We need to reset the controller after the release */ + dev->bsc_regmap->iic_enable = 0; + bsc_writel(dev, dev->bsc_regmap->iic_enable, iic_enable); + + return 0; +} + static int brcmstb_i2c_probe(struct platform_device *pdev) { int rc = 0; @@ -609,6 +634,13 @@ static int brcmstb_i2c_probe(struct platform_device *pdev) goto probe_errorout; } + if (of_device_is_compatible(dev->device->of_node, + "brcm,bcm2711-hdmi-i2c")) { + rc = bcm2711_release_bsc(dev); + if (rc) + goto probe_errorout; + } + rc = of_property_read_string(dev->device->of_node, "interrupt-names", &int_name); if (rc < 0) @@ -705,6 +737,7 @@ static SIMPLE_DEV_PM_OPS(brcmstb_i2c_pm, brcmstb_i2c_suspend, static const struct of_device_id brcmstb_i2c_of_match[] = { {.compatible = "brcm,brcmstb-i2c"}, {.compatible = "brcm,brcmper-i2c"}, + {.compatible = "brcm,bcm2711-hdmi-i2c"}, {}, }; MODULE_DEVICE_TABLE(of, brcmstb_i2c_of_match); diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c index 1105aee6634a..89d58f7d2a25 100644 --- a/drivers/i2c/busses/i2c-cadence.c +++ b/drivers/i2c/busses/i2c-cadence.c @@ -104,9 +104,6 @@ #define DRIVER_NAME "cdns-i2c" -#define CDNS_I2C_SPEED_MAX 400000 -#define CDNS_I2C_SPEED_DEFAULT 100000 - #define CDNS_I2C_DIVA_MAX 4 #define CDNS_I2C_DIVB_MAX 64 @@ -949,8 +946,8 @@ static int cdns_i2c_probe(struct platform_device *pdev) ret = of_property_read_u32(pdev->dev.of_node, "clock-frequency", &id->i2c_clk); - if (ret || (id->i2c_clk > CDNS_I2C_SPEED_MAX)) - id->i2c_clk = CDNS_I2C_SPEED_DEFAULT; + if (ret || (id->i2c_clk > I2C_MAX_FAST_MODE_FREQ)) + id->i2c_clk = I2C_MAX_STANDARD_MODE_FREQ; cdns_i2c_writereg(CDNS_I2C_CR_ACK_EN | CDNS_I2C_CR_NEA | CDNS_I2C_CR_MS, CDNS_I2C_CR_OFFSET); diff --git a/drivers/i2c/busses/i2c-designware-baytrail.c b/drivers/i2c/busses/i2c-designware-baytrail.c index 33da07d64494..c6a7a00e1d52 100644 --- a/drivers/i2c/busses/i2c-designware-baytrail.c +++ b/drivers/i2c/busses/i2c-designware-baytrail.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Intel BayTrail PMIC I2C bus semaphore implementaion + * Intel BayTrail PMIC I2C bus semaphore implementation * Copyright (c) 2014, Intel Corporation. */ #include diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c index 2de7452fcd6d..c70c6fc09ee3 100644 --- a/drivers/i2c/busses/i2c-designware-common.c +++ b/drivers/i2c/busses/i2c-designware-common.c @@ -102,7 +102,7 @@ int i2c_dw_set_reg_access(struct dw_i2c_dev *dev) i2c_dw_release_lock(dev); if (reg == swab32(DW_IC_COMP_TYPE_VALUE)) { - /* Configure register endianess access */ + /* Configure register endianness access */ dev->flags |= ACCESS_SWAP; } else if (reg == (DW_IC_COMP_TYPE_VALUE & 0x0000ffff)) { /* Configure register access mode 16bit */ @@ -190,10 +190,10 @@ int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev) /* * Workaround for avoiding TX arbitration lost in case I2C - * slave pulls SDA down "too quickly" after falling egde of + * slave pulls SDA down "too quickly" after falling edge of * SCL by enabling non-zero SDA RX hold. Specification says it * extends incoming SDA low to high transition while SCL is - * high but it apprears to help also above issue. + * high but it appears to help also above issue. */ if (!(dev->sda_hold_time & DW_IC_SDA_HOLD_RX_MASK)) dev->sda_hold_time |= 1 << DW_IC_SDA_HOLD_RX_SHIFT; @@ -344,6 +344,28 @@ int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev) return -EIO; } +void i2c_dw_set_fifo_size(struct dw_i2c_dev *dev) +{ + u32 param, tx_fifo_depth, rx_fifo_depth; + + /* + * Try to detect the FIFO depth if not set by interface driver, + * the depth could be from 2 to 256 from HW spec. + */ + param = dw_readl(dev, DW_IC_COMP_PARAM_1); + tx_fifo_depth = ((param >> 16) & 0xff) + 1; + rx_fifo_depth = ((param >> 8) & 0xff) + 1; + if (!dev->tx_fifo_depth) { + dev->tx_fifo_depth = tx_fifo_depth; + dev->rx_fifo_depth = rx_fifo_depth; + } else if (tx_fifo_depth >= 2) { + dev->tx_fifo_depth = min_t(u32, dev->tx_fifo_depth, + tx_fifo_depth); + dev->rx_fifo_depth = min_t(u32, dev->rx_fifo_depth, + rx_fifo_depth); + } +} + u32 i2c_dw_func(struct i2c_adapter *adap) { struct dw_i2c_dev *dev = i2c_get_adapdata(adap); @@ -356,7 +378,7 @@ void i2c_dw_disable(struct dw_i2c_dev *dev) /* Disable controller */ __i2c_dw_disable(dev); - /* Disable all interupts */ + /* Disable all interrupts */ dw_writel(dev, 0, DW_IC_INTR_MASK); dw_readl(dev, DW_IC_CLR_INTR); } @@ -366,11 +388,5 @@ void i2c_dw_disable_int(struct dw_i2c_dev *dev) dw_writel(dev, 0, DW_IC_INTR_MASK); } -u32 i2c_dw_read_comp_param(struct dw_i2c_dev *dev) -{ - return dw_readl(dev, DW_IC_COMP_PARAM_1); -} -EXPORT_SYMBOL_GPL(i2c_dw_read_comp_param); - MODULE_DESCRIPTION("Synopsys DesignWare I2C bus adapter core"); MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index 67edbbde1070..b220ad64c38d 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -297,6 +297,7 @@ int i2c_dw_acquire_lock(struct dw_i2c_dev *dev); void i2c_dw_release_lock(struct dw_i2c_dev *dev); int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev); int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev); +void i2c_dw_set_fifo_size(struct dw_i2c_dev *dev); u32 i2c_dw_func(struct i2c_adapter *adap); void i2c_dw_disable(struct dw_i2c_dev *dev); void i2c_dw_disable_int(struct dw_i2c_dev *dev); @@ -313,7 +314,6 @@ static inline void __i2c_dw_disable_nowait(struct dw_i2c_dev *dev) void __i2c_dw_disable(struct dw_i2c_dev *dev); -extern u32 i2c_dw_read_comp_param(struct dw_i2c_dev *dev); extern int i2c_dw_probe(struct dw_i2c_dev *dev); #if IS_ENABLED(CONFIG_I2C_DESIGNWARE_SLAVE) extern int i2c_dw_probe_slave(struct dw_i2c_dev *dev); diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c index e8b328242256..3a58eef20936 100644 --- a/drivers/i2c/busses/i2c-designware-master.c +++ b/drivers/i2c/busses/i2c-designware-master.c @@ -521,7 +521,7 @@ static u32 i2c_dw_read_clear_intrbits(struct dw_i2c_dev *dev) /* * The IC_INTR_STAT register just indicates "enabled" interrupts. - * Ths unmasked raw version of interrupt status bits are available + * The unmasked raw version of interrupt status bits is available * in the IC_RAW_INTR_STAT register. * * That is, @@ -698,6 +698,8 @@ int i2c_dw_probe(struct dw_i2c_dev *dev) if (ret) return ret; + i2c_dw_set_fifo_size(dev); + ret = dev->init(dev); if (ret) return ret; diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c index 05b35ac33ce3..7a0b65b5b5b5 100644 --- a/drivers/i2c/busses/i2c-designware-pcidrv.c +++ b/drivers/i2c/busses/i2c-designware-pcidrv.c @@ -109,7 +109,7 @@ static int mfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c) static int mrfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c) { /* - * On Intel Merrifield the user visible i2c busses are enumerated + * On Intel Merrifield the user visible i2c buses are enumerated * [1..7]. So, we add 1 to shift the default range. Besides that the * first PCI slot provides 4 functions, that's why we have to add 0 to * the first slot and 4 to the next one. diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index 3b7d58c2fe85..c98befe2a92e 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -99,16 +99,16 @@ static int dw_i2c_acpi_configure(struct platform_device *pdev) dw_i2c_acpi_params(pdev, "FMCN", &dev->fs_hcnt, &dev->fs_lcnt, &fs_ht); switch (t->bus_freq_hz) { - case 100000: + case I2C_MAX_STANDARD_MODE_FREQ: dev->sda_hold_time = ss_ht; break; - case 1000000: + case I2C_MAX_FAST_MODE_PLUS_FREQ: dev->sda_hold_time = fp_ht; break; - case 3400000: + case I2C_MAX_HIGH_SPEED_MODE_FREQ: dev->sda_hold_time = hs_ht; break; - case 400000: + case I2C_MAX_FAST_MODE_FREQ: default: dev->sda_hold_time = fs_ht; break; @@ -198,10 +198,10 @@ static void i2c_dw_configure_master(struct dw_i2c_dev *dev) dev->mode = DW_IC_MASTER; switch (t->bus_freq_hz) { - case 100000: + case I2C_MAX_STANDARD_MODE_FREQ: dev->master_cfg |= DW_IC_CON_SPEED_STD; break; - case 3400000: + case I2C_MAX_HIGH_SPEED_MODE_FREQ: dev->master_cfg |= DW_IC_CON_SPEED_HIGH; break; default: @@ -219,28 +219,6 @@ static void i2c_dw_configure_slave(struct dw_i2c_dev *dev) dev->mode = DW_IC_SLAVE; } -static void dw_i2c_set_fifo_size(struct dw_i2c_dev *dev) -{ - u32 param, tx_fifo_depth, rx_fifo_depth; - - /* - * Try to detect the FIFO depth if not set by interface driver, - * the depth could be from 2 to 256 from HW spec. - */ - param = i2c_dw_read_comp_param(dev); - tx_fifo_depth = ((param >> 16) & 0xff) + 1; - rx_fifo_depth = ((param >> 8) & 0xff) + 1; - if (!dev->tx_fifo_depth) { - dev->tx_fifo_depth = tx_fifo_depth; - dev->rx_fifo_depth = rx_fifo_depth; - } else if (tx_fifo_depth >= 2) { - dev->tx_fifo_depth = min_t(u32, dev->tx_fifo_depth, - tx_fifo_depth); - dev->rx_fifo_depth = min_t(u32, dev->rx_fifo_depth, - rx_fifo_depth); - } -} - static void dw_i2c_plat_pm_cleanup(struct dw_i2c_dev *dev) { pm_runtime_disable(dev->dev); @@ -249,6 +227,13 @@ static void dw_i2c_plat_pm_cleanup(struct dw_i2c_dev *dev) pm_runtime_put_noidle(dev->dev); } +static const u32 supported_speeds[] = { + I2C_MAX_HIGH_SPEED_MODE_FREQ, + I2C_MAX_FAST_MODE_PLUS_FREQ, + I2C_MAX_FAST_MODE_FREQ, + I2C_MAX_STANDARD_MODE_FREQ, +}; + static int dw_i2c_plat_probe(struct platform_device *pdev) { struct dw_i2c_platform_data *pdata = dev_get_platdata(&pdev->dev); @@ -258,9 +243,6 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) u32 acpi_speed; struct resource *mem; int i, irq, ret; - static const int supported_speeds[] = { - 0, 100000, 400000, 1000000, 3400000 - }; irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -296,11 +278,11 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) * Some DSTDs use a non standard speed, round down to the lowest * standard speed. */ - for (i = 1; i < ARRAY_SIZE(supported_speeds); i++) { - if (acpi_speed < supported_speeds[i]) + for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) { + if (acpi_speed >= supported_speeds[i]) break; } - acpi_speed = supported_speeds[i - 1]; + acpi_speed = i < ARRAY_SIZE(supported_speeds) ? supported_speeds[i] : 0; /* * Find bus speed from the "clock-frequency" device property, ACPI @@ -311,7 +293,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) else if (acpi_speed || t->bus_freq_hz) t->bus_freq_hz = max(t->bus_freq_hz, acpi_speed); else - t->bus_freq_hz = 400000; + t->bus_freq_hz = I2C_MAX_FAST_MODE_FREQ; dev->flags |= (uintptr_t)device_get_match_data(&pdev->dev); @@ -325,8 +307,11 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) * Only standard mode at 100kHz, fast mode at 400kHz, * fast mode plus at 1MHz and high speed mode at 3.4MHz are supported. */ - if (t->bus_freq_hz != 100000 && t->bus_freq_hz != 400000 && - t->bus_freq_hz != 1000000 && t->bus_freq_hz != 3400000) { + for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) { + if (t->bus_freq_hz == supported_speeds[i]) + break; + } + if (i == ARRAY_SIZE(supported_speeds)) { dev_err(&pdev->dev, "%d Hz is unsupported, only 100kHz, 400kHz, 1MHz and 3.4MHz are supported\n", t->bus_freq_hz); @@ -362,8 +347,6 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) div_u64(clk_khz * t->sda_hold_ns + 500000, 1000000); } - dw_i2c_set_fifo_size(dev); - adap = &dev->adapter; adap->owner = THIS_MODULE; adap->class = I2C_CLASS_DEPRECATED; diff --git a/drivers/i2c/busses/i2c-designware-slave.c b/drivers/i2c/busses/i2c-designware-slave.c index f5f001738df5..f5ecf76c0d02 100644 --- a/drivers/i2c/busses/i2c-designware-slave.c +++ b/drivers/i2c/busses/i2c-designware-slave.c @@ -107,7 +107,7 @@ static u32 i2c_dw_read_clear_intrbits_slave(struct dw_i2c_dev *dev) /* * The IC_INTR_STAT register just indicates "enabled" interrupts. - * Ths unmasked raw version of interrupt status bits are available + * The unmasked raw version of interrupt status bits is available * in the IC_RAW_INTR_STAT register. * * That is, @@ -260,6 +260,8 @@ int i2c_dw_probe_slave(struct dw_i2c_dev *dev) if (ret) return ret; + i2c_dw_set_fifo_size(dev); + ret = dev->init(dev); if (ret) return ret; diff --git a/drivers/i2c/busses/i2c-digicolor.c b/drivers/i2c/busses/i2c-digicolor.c index 3adf72540db1..056a5c4f0833 100644 --- a/drivers/i2c/busses/i2c-digicolor.c +++ b/drivers/i2c/busses/i2c-digicolor.c @@ -18,7 +18,6 @@ #include #include -#define DEFAULT_FREQ 100000 #define TIMEOUT_MS 100 #define II_CONTROL 0x0 @@ -300,7 +299,7 @@ static int dc_i2c_probe(struct platform_device *pdev) if (of_property_read_u32(pdev->dev.of_node, "clock-frequency", &i2c->frequency)) - i2c->frequency = DEFAULT_FREQ; + i2c->frequency = I2C_MAX_STANDARD_MODE_FREQ; i2c->dev = &pdev->dev; platform_set_drvdata(pdev, i2c); diff --git a/drivers/i2c/busses/i2c-diolan-u2c.c b/drivers/i2c/busses/i2c-diolan-u2c.c index 382f105e0fe3..b48b7888936f 100644 --- a/drivers/i2c/busses/i2c-diolan-u2c.c +++ b/drivers/i2c/busses/i2c-diolan-u2c.c @@ -64,8 +64,6 @@ #define U2C_I2C_SPEED_2KHZ 242 /* 2 kHz, minimum speed */ #define U2C_I2C_SPEED(f) ((DIV_ROUND_UP(1000000, (f)) - 10) / 2 + 1) -#define U2C_I2C_FREQ_FAST 400000 -#define U2C_I2C_FREQ_STD 100000 #define U2C_I2C_FREQ(s) (1000000 / (2 * (s - 1) + 10)) #define DIOLAN_USB_TIMEOUT 100 /* in ms */ @@ -87,7 +85,7 @@ struct i2c_diolan_u2c { int ocount; /* Number of enqueued messages */ }; -static uint frequency = U2C_I2C_FREQ_STD; /* I2C clock frequency in Hz */ +static uint frequency = I2C_MAX_STANDARD_MODE_FREQ; /* I2C clock frequency in Hz */ module_param(frequency, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(frequency, "I2C clock frequency in hertz"); @@ -299,12 +297,12 @@ static int diolan_init(struct i2c_diolan_u2c *dev) { int speed, ret; - if (frequency >= 200000) { + if (frequency >= 2 * I2C_MAX_STANDARD_MODE_FREQ) { speed = U2C_I2C_SPEED_FAST; - frequency = U2C_I2C_FREQ_FAST; - } else if (frequency >= 100000 || frequency == 0) { + frequency = I2C_MAX_FAST_MODE_FREQ; + } else if (frequency >= I2C_MAX_STANDARD_MODE_FREQ || frequency == 0) { speed = U2C_I2C_SPEED_STD; - frequency = U2C_I2C_FREQ_STD; + frequency = I2C_MAX_STANDARD_MODE_FREQ; } else { speed = U2C_I2C_SPEED(frequency); if (speed > U2C_I2C_SPEED_2KHZ) diff --git a/drivers/i2c/busses/i2c-efm32.c b/drivers/i2c/busses/i2c-efm32.c index a8c6323e7f44..18cca8f56da8 100644 --- a/drivers/i2c/busses/i2c-efm32.c +++ b/drivers/i2c/busses/i2c-efm32.c @@ -388,7 +388,7 @@ static int efm32_i2c_probe(struct platform_device *pdev) if (!ret) { dev_dbg(&pdev->dev, "using frequency %u\n", frequency); } else { - frequency = 100000; + frequency = I2C_MAX_STANDARD_MODE_FREQ; dev_info(&pdev->dev, "defaulting to 100 kHz\n"); } ddata->frequency = frequency; diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c index e7514c16b756..527030953ba1 100644 --- a/drivers/i2c/busses/i2c-exynos5.c +++ b/drivers/i2c/busses/i2c-exynos5.c @@ -164,13 +164,6 @@ #define HSI2C_MASTER_ID(x) ((x & 0xff) << 24) #define MASTER_ID(x) ((x & 0x7) + 0x08) -/* - * Controller operating frequency, timing values for operation - * are calculated against this frequency - */ -#define HSI2C_HS_TX_CLOCK 1000000 -#define HSI2C_FS_TX_CLOCK 100000 - #define EXYNOS5_I2C_TIMEOUT (msecs_to_jiffies(100)) enum i2c_type_exynos { @@ -264,6 +257,9 @@ static void exynos5_i2c_clr_pend_irq(struct exynos5_i2c *i2c) * exynos5_i2c_set_timing: updates the registers with appropriate * timing values calculated * + * Timing values for operation are calculated against either 100kHz + * or 1MHz controller operating frequency. + * * Returns 0 on success, -EINVAL if the cycle length cannot * be calculated. */ @@ -281,7 +277,7 @@ static int exynos5_i2c_set_timing(struct exynos5_i2c *i2c, bool hs_timings) unsigned int t_ftl_cycle; unsigned int clkin = clk_get_rate(i2c->clk); unsigned int op_clk = hs_timings ? i2c->op_clock : - (i2c->op_clock >= HSI2C_HS_TX_CLOCK) ? HSI2C_FS_TX_CLOCK : + (i2c->op_clock >= I2C_MAX_FAST_MODE_PLUS_FREQ) ? I2C_MAX_STANDARD_MODE_FREQ : i2c->op_clock; int div, clk_cycle, temp; @@ -353,7 +349,7 @@ static int exynos5_hsi2c_clock_setup(struct exynos5_i2c *i2c) /* always set Fast Speed timings */ int ret = exynos5_i2c_set_timing(i2c, false); - if (ret < 0 || i2c->op_clock < HSI2C_HS_TX_CLOCK) + if (ret < 0 || i2c->op_clock < I2C_MAX_FAST_MODE_PLUS_FREQ) return ret; return exynos5_i2c_set_timing(i2c, true); @@ -376,7 +372,7 @@ static void exynos5_i2c_init(struct exynos5_i2c *i2c) i2c->regs + HSI2C_CTL); writel(HSI2C_TRAILING_COUNT, i2c->regs + HSI2C_TRAILIG_CTL); - if (i2c->op_clock >= HSI2C_HS_TX_CLOCK) { + if (i2c->op_clock >= I2C_MAX_FAST_MODE_PLUS_FREQ) { writel(HSI2C_MASTER_ID(MASTER_ID(i2c->adap.nr)), i2c->regs + HSI2C_ADDR); i2c_conf |= HSI2C_HS_MODE; @@ -748,7 +744,7 @@ static int exynos5_i2c_probe(struct platform_device *pdev) return -ENOMEM; if (of_property_read_u32(np, "clock-frequency", &i2c->op_clock)) - i2c->op_clock = HSI2C_FS_TX_CLOCK; + i2c->op_clock = I2C_MAX_STANDARD_MODE_FREQ; strlcpy(i2c->adap.name, "exynos5-i2c", sizeof(i2c->adap.name)); i2c->adap.owner = THIS_MODULE; diff --git a/drivers/i2c/busses/i2c-hix5hd2.c b/drivers/i2c/busses/i2c-hix5hd2.c index 224f830f77f9..6610304b6dc6 100644 --- a/drivers/i2c/busses/i2c-hix5hd2.c +++ b/drivers/i2c/busses/i2c-hix5hd2.c @@ -68,8 +68,6 @@ #define I2C_ARBITRATE_INTR BIT(1) #define I2C_OVER_INTR BIT(0) -#define HIX5I2C_MAX_FREQ 400000 /* 400k */ - enum hix5hd2_i2c_state { HIX5I2C_STAT_RW_ERR = -1, HIX5I2C_STAT_INIT, @@ -400,12 +398,12 @@ static int hix5hd2_i2c_probe(struct platform_device *pdev) if (of_property_read_u32(np, "clock-frequency", &freq)) { /* use 100k as default value */ - priv->freq = 100000; + priv->freq = I2C_MAX_STANDARD_MODE_FREQ; } else { - if (freq > HIX5I2C_MAX_FREQ) { - priv->freq = HIX5I2C_MAX_FREQ; + if (freq > I2C_MAX_FAST_MODE_FREQ) { + priv->freq = I2C_MAX_FAST_MODE_FREQ; dev_warn(priv->dev, "use max freq %d instead\n", - HIX5I2C_MAX_FREQ); + I2C_MAX_FAST_MODE_FREQ); } else { priv->freq = freq; } diff --git a/drivers/i2c/busses/i2c-img-scb.c b/drivers/i2c/busses/i2c-img-scb.c index 20a4fbc53007..422097a31c95 100644 --- a/drivers/i2c/busses/i2c-img-scb.c +++ b/drivers/i2c/busses/i2c-img-scb.c @@ -304,7 +304,7 @@ static struct img_i2c_timings timings[] = { /* Standard mode */ { .name = "standard", - .max_bitrate = 100000, + .max_bitrate = I2C_MAX_STANDARD_MODE_FREQ, .tckh = 4000, .tckl = 4700, .tsdh = 4700, @@ -316,7 +316,7 @@ static struct img_i2c_timings timings[] = { /* Fast mode */ { .name = "fast", - .max_bitrate = 400000, + .max_bitrate = I2C_MAX_FAST_MODE_FREQ, .tckh = 600, .tckl = 1300, .tsdh = 600, diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index c92b56485fa6..94743ba581fe 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -75,12 +75,6 @@ #define I2C_CLK_RATIO 2 #define CHUNK_DATA 256 -#define LPI2C_DEFAULT_RATE 100000 -#define STARDARD_MAX_BITRATE 400000 -#define FAST_MAX_BITRATE 1000000 -#define FAST_PLUS_MAX_BITRATE 3400000 -#define HIGHSPEED_MAX_BITRATE 5000000 - #define I2C_PM_TIMEOUT 10 /* ms */ enum lpi2c_imx_mode { @@ -152,13 +146,13 @@ static void lpi2c_imx_set_mode(struct lpi2c_imx_struct *lpi2c_imx) unsigned int bitrate = lpi2c_imx->bitrate; enum lpi2c_imx_mode mode; - if (bitrate < STARDARD_MAX_BITRATE) + if (bitrate < I2C_MAX_FAST_MODE_FREQ) mode = STANDARD; - else if (bitrate < FAST_MAX_BITRATE) + else if (bitrate < I2C_MAX_FAST_MODE_PLUS_FREQ) mode = FAST; - else if (bitrate < FAST_PLUS_MAX_BITRATE) + else if (bitrate < I2C_MAX_HIGH_SPEED_MODE_FREQ) mode = FAST_PLUS; - else if (bitrate < HIGHSPEED_MAX_BITRATE) + else if (bitrate < I2C_MAX_ULTRA_FAST_MODE_FREQ) mode = HS; else mode = ULTRA_FAST; @@ -578,7 +572,7 @@ static int lpi2c_imx_probe(struct platform_device *pdev) ret = of_property_read_u32(pdev->dev.of_node, "clock-frequency", &lpi2c_imx->bitrate); if (ret) - lpi2c_imx->bitrate = LPI2C_DEFAULT_RATE; + lpi2c_imx->bitrate = I2C_MAX_STANDARD_MODE_FREQ; ret = devm_request_irq(&pdev->dev, irq, lpi2c_imx_isr, 0, pdev->name, lpi2c_imx); diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index a3b61336fe55..0ab5381aa012 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -49,9 +50,6 @@ /* This will be the driver name the kernel reports */ #define DRIVER_NAME "imx-i2c" -/* Default value */ -#define IMX_I2C_BIT_RATE 100000 /* 100kHz */ - /* * Enable DMA if transfer byte size is bigger than this threshold. * As the hardware request, it must bigger than 4 bytes.\ @@ -414,7 +412,7 @@ static void i2c_imx_dma_free(struct imx_i2c_struct *i2c_imx) dma->chan_using = NULL; } -static int i2c_imx_bus_busy(struct imx_i2c_struct *i2c_imx, int for_busy) +static int i2c_imx_bus_busy(struct imx_i2c_struct *i2c_imx, int for_busy, bool atomic) { unsigned long orig_jiffies = jiffies; unsigned int temp; @@ -444,15 +442,37 @@ static int i2c_imx_bus_busy(struct imx_i2c_struct *i2c_imx, int for_busy) "<%s> I2C bus is busy\n", __func__); return -ETIMEDOUT; } - schedule(); + if (atomic) + udelay(100); + else + schedule(); } return 0; } -static int i2c_imx_trx_complete(struct imx_i2c_struct *i2c_imx) +static int i2c_imx_trx_complete(struct imx_i2c_struct *i2c_imx, bool atomic) { - wait_event_timeout(i2c_imx->queue, i2c_imx->i2csr & I2SR_IIF, HZ / 10); + if (atomic) { + void __iomem *addr = i2c_imx->base + (IMX_I2C_I2SR << i2c_imx->hwdata->regshift); + unsigned int regval; + + /* + * The formula for the poll timeout is documented in the RM + * Rev.5 on page 1878: + * T_min = 10/F_scl + * Set the value hard as it is done for the non-atomic use-case. + * Use 10 kHz for the calculation since this is the minimum + * allowed SMBus frequency. Also add an offset of 100us since it + * turned out that the I2SR_IIF bit isn't set correctly within + * the minimum timeout in polling mode. + */ + readb_poll_timeout_atomic(addr, regval, regval & I2SR_IIF, 5, 1000 + 100); + i2c_imx->i2csr = regval; + imx_i2c_write_reg(0, i2c_imx, IMX_I2C_I2SR); + } else { + wait_event_timeout(i2c_imx->queue, i2c_imx->i2csr & I2SR_IIF, HZ / 10); + } if (unlikely(!(i2c_imx->i2csr & I2SR_IIF))) { dev_dbg(&i2c_imx->adapter.dev, "<%s> Timeout\n", __func__); @@ -530,7 +550,7 @@ static int i2c_imx_clk_notifier_call(struct notifier_block *nb, return NOTIFY_OK; } -static int i2c_imx_start(struct imx_i2c_struct *i2c_imx) +static int i2c_imx_start(struct imx_i2c_struct *i2c_imx, bool atomic) { unsigned int temp = 0; int result; @@ -543,23 +563,29 @@ static int i2c_imx_start(struct imx_i2c_struct *i2c_imx) imx_i2c_write_reg(i2c_imx->hwdata->i2cr_ien_opcode, i2c_imx, IMX_I2C_I2CR); /* Wait controller to be stable */ - usleep_range(50, 150); + if (atomic) + udelay(50); + else + usleep_range(50, 150); /* Start I2C transaction */ temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); temp |= I2CR_MSTA; imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); - result = i2c_imx_bus_busy(i2c_imx, 1); + result = i2c_imx_bus_busy(i2c_imx, 1, atomic); if (result) return result; temp |= I2CR_IIEN | I2CR_MTX | I2CR_TXAK; + if (atomic) + temp &= ~I2CR_IIEN; /* Disable interrupt */ + temp &= ~I2CR_DMAEN; imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); return result; } -static void i2c_imx_stop(struct imx_i2c_struct *i2c_imx) +static void i2c_imx_stop(struct imx_i2c_struct *i2c_imx, bool atomic) { unsigned int temp = 0; @@ -581,7 +607,7 @@ static void i2c_imx_stop(struct imx_i2c_struct *i2c_imx) } if (!i2c_imx->stopped) - i2c_imx_bus_busy(i2c_imx, 0); + i2c_imx_bus_busy(i2c_imx, 0, atomic); /* Disable I2C controller */ temp = i2c_imx->hwdata->i2cr_ien_opcode ^ I2CR_IEN, @@ -662,7 +688,7 @@ static int i2c_imx_dma_write(struct imx_i2c_struct *i2c_imx, /* The last data byte must be transferred by the CPU. */ imx_i2c_write_reg(msgs->buf[msgs->len-1], i2c_imx, IMX_I2C_I2DR); - result = i2c_imx_trx_complete(i2c_imx); + result = i2c_imx_trx_complete(i2c_imx, false); if (result) return result; @@ -721,7 +747,7 @@ static int i2c_imx_dma_read(struct imx_i2c_struct *i2c_imx, msgs->buf[msgs->len-2] = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR); /* read n byte data */ - result = i2c_imx_trx_complete(i2c_imx); + result = i2c_imx_trx_complete(i2c_imx, false); if (result) return result; @@ -734,7 +760,7 @@ static int i2c_imx_dma_read(struct imx_i2c_struct *i2c_imx, temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); temp &= ~(I2CR_MSTA | I2CR_MTX); imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); - i2c_imx_bus_busy(i2c_imx, 0); + i2c_imx_bus_busy(i2c_imx, 0, false); } else { /* * For i2c master receiver repeat restart operation like: @@ -752,7 +778,8 @@ static int i2c_imx_dma_read(struct imx_i2c_struct *i2c_imx, return 0; } -static int i2c_imx_write(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs) +static int i2c_imx_write(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, + bool atomic) { int i, result; @@ -761,7 +788,7 @@ static int i2c_imx_write(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs) /* write slave address */ imx_i2c_write_reg(i2c_8bit_addr_from_msg(msgs), i2c_imx, IMX_I2C_I2DR); - result = i2c_imx_trx_complete(i2c_imx); + result = i2c_imx_trx_complete(i2c_imx, atomic); if (result) return result; result = i2c_imx_acked(i2c_imx); @@ -775,7 +802,7 @@ static int i2c_imx_write(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs) "<%s> write byte: B%d=0x%X\n", __func__, i, msgs->buf[i]); imx_i2c_write_reg(msgs->buf[i], i2c_imx, IMX_I2C_I2DR); - result = i2c_imx_trx_complete(i2c_imx); + result = i2c_imx_trx_complete(i2c_imx, atomic); if (result) return result; result = i2c_imx_acked(i2c_imx); @@ -785,7 +812,8 @@ static int i2c_imx_write(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs) return 0; } -static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, bool is_lastmsg) +static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, + bool is_lastmsg, bool atomic) { int i, result; unsigned int temp; @@ -798,7 +826,7 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, bo /* write slave address */ imx_i2c_write_reg(i2c_8bit_addr_from_msg(msgs), i2c_imx, IMX_I2C_I2DR); - result = i2c_imx_trx_complete(i2c_imx); + result = i2c_imx_trx_complete(i2c_imx, atomic); if (result) return result; result = i2c_imx_acked(i2c_imx); @@ -831,7 +859,7 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, bo for (i = 0; i < msgs->len; i++) { u8 len = 0; - result = i2c_imx_trx_complete(i2c_imx); + result = i2c_imx_trx_complete(i2c_imx, atomic); if (result) return result; /* @@ -859,7 +887,7 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, bo temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); temp &= ~(I2CR_MSTA | I2CR_MTX); imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); - i2c_imx_bus_busy(i2c_imx, 0); + i2c_imx_bus_busy(i2c_imx, 0, atomic); } else { /* * For i2c master receiver repeat restart operation like: @@ -890,8 +918,8 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, bo return 0; } -static int i2c_imx_xfer(struct i2c_adapter *adapter, - struct i2c_msg *msgs, int num) +static int i2c_imx_xfer_common(struct i2c_adapter *adapter, + struct i2c_msg *msgs, int num, bool atomic) { unsigned int i, temp; int result; @@ -900,16 +928,16 @@ static int i2c_imx_xfer(struct i2c_adapter *adapter, dev_dbg(&i2c_imx->adapter.dev, "<%s>\n", __func__); - result = pm_runtime_get_sync(i2c_imx->adapter.dev.parent); - if (result < 0) - goto out; - /* Start I2C transfer */ - result = i2c_imx_start(i2c_imx); + result = i2c_imx_start(i2c_imx, atomic); if (result) { - if (i2c_imx->adapter.bus_recovery_info) { + /* + * Bus recovery uses gpiod_get_value_cansleep() which is not + * allowed within atomic context. + */ + if (!atomic && i2c_imx->adapter.bus_recovery_info) { i2c_recover_bus(&i2c_imx->adapter); - result = i2c_imx_start(i2c_imx); + result = i2c_imx_start(i2c_imx, atomic); } } @@ -927,7 +955,7 @@ static int i2c_imx_xfer(struct i2c_adapter *adapter, temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); temp |= I2CR_RSTA; imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); - result = i2c_imx_bus_busy(i2c_imx, 1); + result = i2c_imx_bus_busy(i2c_imx, 1, atomic); if (result) goto fail0; } @@ -951,13 +979,14 @@ static int i2c_imx_xfer(struct i2c_adapter *adapter, (temp & I2SR_SRW ? 1 : 0), (temp & I2SR_IIF ? 1 : 0), (temp & I2SR_RXAK ? 1 : 0)); #endif - if (msgs[i].flags & I2C_M_RD) - result = i2c_imx_read(i2c_imx, &msgs[i], is_lastmsg); - else { - if (i2c_imx->dma && msgs[i].len >= DMA_THRESHOLD) + if (msgs[i].flags & I2C_M_RD) { + result = i2c_imx_read(i2c_imx, &msgs[i], is_lastmsg, atomic); + } else { + if (!atomic && + i2c_imx->dma && msgs[i].len >= DMA_THRESHOLD) result = i2c_imx_dma_write(i2c_imx, &msgs[i]); else - result = i2c_imx_write(i2c_imx, &msgs[i]); + result = i2c_imx_write(i2c_imx, &msgs[i], atomic); } if (result) goto fail0; @@ -965,18 +994,49 @@ static int i2c_imx_xfer(struct i2c_adapter *adapter, fail0: /* Stop I2C transfer */ - i2c_imx_stop(i2c_imx); + i2c_imx_stop(i2c_imx, atomic); - pm_runtime_mark_last_busy(i2c_imx->adapter.dev.parent); - pm_runtime_put_autosuspend(i2c_imx->adapter.dev.parent); - -out: dev_dbg(&i2c_imx->adapter.dev, "<%s> exit with: %s: %d\n", __func__, (result < 0) ? "error" : "success msg", (result < 0) ? result : num); return (result < 0) ? result : num; } +static int i2c_imx_xfer(struct i2c_adapter *adapter, + struct i2c_msg *msgs, int num) +{ + struct imx_i2c_struct *i2c_imx = i2c_get_adapdata(adapter); + int result; + + result = pm_runtime_get_sync(i2c_imx->adapter.dev.parent); + if (result < 0) + return result; + + result = i2c_imx_xfer_common(adapter, msgs, num, false); + + pm_runtime_mark_last_busy(i2c_imx->adapter.dev.parent); + pm_runtime_put_autosuspend(i2c_imx->adapter.dev.parent); + + return result; +} + +static int i2c_imx_xfer_atomic(struct i2c_adapter *adapter, + struct i2c_msg *msgs, int num) +{ + struct imx_i2c_struct *i2c_imx = i2c_get_adapdata(adapter); + int result; + + result = clk_enable(i2c_imx->clk); + if (result) + return result; + + result = i2c_imx_xfer_common(adapter, msgs, num, true); + + clk_disable(i2c_imx->clk); + + return result; +} + static void i2c_imx_prepare_recovery(struct i2c_adapter *adap) { struct imx_i2c_struct *i2c_imx; @@ -1049,8 +1109,9 @@ static u32 i2c_imx_func(struct i2c_adapter *adapter) } static const struct i2c_algorithm i2c_imx_algo = { - .master_xfer = i2c_imx_xfer, - .functionality = i2c_imx_func, + .master_xfer = i2c_imx_xfer, + .master_xfer_atomic = i2c_imx_xfer_atomic, + .functionality = i2c_imx_func, }; static int i2c_imx_probe(struct platform_device *pdev) @@ -1066,10 +1127,8 @@ static int i2c_imx_probe(struct platform_device *pdev) dev_dbg(&pdev->dev, "<%s>\n", __func__); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "can't get irq number\n"); + if (irq < 0) return irq; - } res = platform_get_resource(pdev, IORESOURCE_MEM, 0); base = devm_ioremap_resource(&pdev->dev, res); @@ -1139,7 +1198,7 @@ static int i2c_imx_probe(struct platform_device *pdev) goto rpm_disable; /* Set up clock divider */ - i2c_imx->bitrate = IMX_I2C_BIT_RATE; + i2c_imx->bitrate = I2C_MAX_STANDARD_MODE_FREQ; ret = of_property_read_u32(pdev->dev.of_node, "clock-frequency", &i2c_imx->bitrate); if (ret < 0 && pdata && pdata->bitrate) diff --git a/drivers/i2c/busses/i2c-lpc2k.c b/drivers/i2c/busses/i2c-lpc2k.c index deea18b14add..13b0c12e2dba 100644 --- a/drivers/i2c/busses/i2c-lpc2k.c +++ b/drivers/i2c/busses/i2c-lpc2k.c @@ -396,7 +396,7 @@ static int i2c_lpc2k_probe(struct platform_device *pdev) ret = of_property_read_u32(pdev->dev.of_node, "clock-frequency", &bus_clk_rate); if (ret) - bus_clk_rate = 100000; /* 100 kHz default clock rate */ + bus_clk_rate = I2C_MAX_STANDARD_MODE_FREQ; clkrate = clk_get_rate(i2c->clk); if (clkrate == 0) { @@ -407,9 +407,9 @@ static int i2c_lpc2k_probe(struct platform_device *pdev) /* Setup I2C dividers to generate clock with proper duty cycle */ clkrate = clkrate / bus_clk_rate; - if (bus_clk_rate <= 100000) + if (bus_clk_rate <= I2C_MAX_STANDARD_MODE_FREQ) scl_high = (clkrate * I2C_STD_MODE_DUTY) / 100; - else if (bus_clk_rate <= 400000) + else if (bus_clk_rate <= I2C_MAX_FAST_MODE_FREQ) scl_high = (clkrate * I2C_FAST_MODE_DUTY) / 100; else scl_high = (clkrate * I2C_FAST_MODE_PLUS_DUTY) / 100; diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c index 2152ec5f535c..0ca6c38a15eb 100644 --- a/drivers/i2c/busses/i2c-mt65xx.c +++ b/drivers/i2c/busses/i2c-mt65xx.c @@ -56,9 +56,6 @@ #define I2C_DMA_4G_MODE 0x0001 #define I2C_DEFAULT_CLK_DIV 5 -#define I2C_DEFAULT_SPEED 100000 /* hz */ -#define MAX_FS_MODE_SPEED 400000 -#define MAX_HS_MODE_SPEED 3400000 #define MAX_SAMPLE_CNT_DIV 8 #define MAX_STEP_CNT_DIV 64 #define MAX_HS_STEP_CNT_DIV 8 @@ -450,10 +447,10 @@ static int mtk_i2c_calculate_speed(struct mtk_i2c *i2c, unsigned int clk_src, unsigned int best_mul; unsigned int cnt_mul; - if (target_speed > MAX_HS_MODE_SPEED) - target_speed = MAX_HS_MODE_SPEED; + if (target_speed > I2C_MAX_FAST_MODE_PLUS_FREQ) + target_speed = I2C_MAX_FAST_MODE_PLUS_FREQ; - if (target_speed > MAX_FS_MODE_SPEED) + if (target_speed > I2C_MAX_FAST_MODE_FREQ) max_step_cnt = MAX_HS_STEP_CNT_DIV; else max_step_cnt = MAX_STEP_CNT_DIV; @@ -514,9 +511,9 @@ static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk) clk_src = parent_clk / i2c->clk_src_div; target_speed = i2c->speed_hz; - if (target_speed > MAX_FS_MODE_SPEED) { + if (target_speed > I2C_MAX_FAST_MODE_FREQ) { /* Set master code speed register */ - ret = mtk_i2c_calculate_speed(i2c, clk_src, MAX_FS_MODE_SPEED, + ret = mtk_i2c_calculate_speed(i2c, clk_src, I2C_MAX_FAST_MODE_FREQ, &l_step_cnt, &l_sample_cnt); if (ret < 0) return ret; @@ -581,7 +578,7 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs, control_reg = mtk_i2c_readw(i2c, OFFSET_CONTROL) & ~(I2C_CONTROL_DIR_CHANGE | I2C_CONTROL_RS); - if ((i2c->speed_hz > MAX_FS_MODE_SPEED) || (left_num >= 1)) + if ((i2c->speed_hz > I2C_MAX_FAST_MODE_FREQ) || (left_num >= 1)) control_reg |= I2C_CONTROL_RS; if (i2c->op == I2C_MASTER_WRRD) @@ -590,7 +587,7 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs, mtk_i2c_writew(i2c, control_reg, OFFSET_CONTROL); /* set start condition */ - if (i2c->speed_hz <= I2C_DEFAULT_SPEED) + if (i2c->speed_hz <= I2C_MAX_STANDARD_MODE_FREQ) mtk_i2c_writew(i2c, I2C_ST_START_CON, OFFSET_EXT_CONF); else mtk_i2c_writew(i2c, I2C_FS_START_CON, OFFSET_EXT_CONF); @@ -798,7 +795,7 @@ static int mtk_i2c_transfer(struct i2c_adapter *adap, } } - if (i2c->auto_restart && num >= 2 && i2c->speed_hz > MAX_FS_MODE_SPEED) + if (i2c->auto_restart && num >= 2 && i2c->speed_hz > I2C_MAX_FAST_MODE_FREQ) /* ignore the first restart irq after the master code, * otherwise the first transfer will be discarded. */ @@ -893,7 +890,7 @@ static int mtk_i2c_parse_dt(struct device_node *np, struct mtk_i2c *i2c) ret = of_property_read_u32(np, "clock-frequency", &i2c->speed_hz); if (ret < 0) - i2c->speed_hz = I2C_DEFAULT_SPEED; + i2c->speed_hz = I2C_MAX_STANDARD_MODE_FREQ; ret = of_property_read_u32(np, "clock-div", &i2c->clk_src_div); if (ret < 0) diff --git a/drivers/i2c/busses/i2c-mt7621.c b/drivers/i2c/busses/i2c-mt7621.c index 62df8379bc89..45fe4a7fe0c0 100644 --- a/drivers/i2c/busses/i2c-mt7621.c +++ b/drivers/i2c/busses/i2c-mt7621.c @@ -300,7 +300,7 @@ static int mtk_i2c_probe(struct platform_device *pdev) if (of_property_read_u32(pdev->dev.of_node, "clock-frequency", &i2c->bus_freq)) - i2c->bus_freq = 100000; + i2c->bus_freq = I2C_MAX_STANDARD_MODE_FREQ; if (i2c->bus_freq == 0) { dev_warn(i2c->dev, "clock-frequency 0 not supported\n"); diff --git a/drivers/i2c/busses/i2c-mv64xxx.c b/drivers/i2c/busses/i2c-mv64xxx.c index febb7c7ea72b..9b8f1d8552ea 100644 --- a/drivers/i2c/busses/i2c-mv64xxx.c +++ b/drivers/i2c/busses/i2c-mv64xxx.c @@ -810,7 +810,7 @@ mv64xxx_of_config(struct mv64xxx_i2c_data *drv_data, tclk = clk_get_rate(drv_data->clk); if (of_property_read_u32(np, "clock-frequency", &bus_freq)) - bus_freq = 100000; /* 100kHz by default */ + bus_freq = I2C_MAX_STANDARD_MODE_FREQ; /* 100kHz by default */ if (of_device_is_compatible(np, "allwinner,sun4i-a10-i2c") || of_device_is_compatible(np, "allwinner,sun6i-a31-i2c")) @@ -846,14 +846,14 @@ mv64xxx_of_config(struct mv64xxx_i2c_data *drv_data, if (of_device_is_compatible(np, "marvell,mv78230-i2c")) { drv_data->offload_enabled = true; /* The delay is only needed in standard mode (100kHz) */ - if (bus_freq <= 100000) + if (bus_freq <= I2C_MAX_STANDARD_MODE_FREQ) drv_data->errata_delay = true; } if (of_device_is_compatible(np, "marvell,mv78230-a0-i2c")) { drv_data->offload_enabled = false; /* The delay is only needed in standard mode (100kHz) */ - if (bus_freq <= 100000) + if (bus_freq <= I2C_MAX_STANDARD_MODE_FREQ) drv_data->errata_delay = true; } diff --git a/drivers/i2c/busses/i2c-mxs.c b/drivers/i2c/busses/i2c-mxs.c index 89224913f578..9587347447f0 100644 --- a/drivers/i2c/busses/i2c-mxs.c +++ b/drivers/i2c/busses/i2c-mxs.c @@ -731,7 +731,7 @@ static void mxs_i2c_derive_timing(struct mxs_i2c_dev *i2c, uint32_t speed) * This is compensated for by subtracting the respective constants * from the values written to the timing registers. */ - if (speed > 100000) { + if (speed > I2C_MAX_STANDARD_MODE_FREQ) { /* fast mode */ low_count = DIV_ROUND_CLOSEST(divider * 13, (13 + 6)); high_count = DIV_ROUND_CLOSEST(divider * 6, (13 + 6)); @@ -769,7 +769,7 @@ static int mxs_i2c_get_ofdata(struct mxs_i2c_dev *i2c) ret = of_property_read_u32(node, "clock-frequency", &speed); if (ret) { dev_warn(dev, "No I2C speed selected, using 100kHz\n"); - speed = 100000; + speed = I2C_MAX_STANDARD_MODE_FREQ; } mxs_i2c_derive_timing(i2c, speed); @@ -836,10 +836,10 @@ static int mxs_i2c_probe(struct platform_device *pdev) } /* Setup the DMA */ - i2c->dmach = dma_request_slave_channel(dev, "rx-tx"); - if (!i2c->dmach) { + i2c->dmach = dma_request_chan(dev, "rx-tx"); + if (IS_ERR(i2c->dmach)) { dev_err(dev, "Failed to request dma\n"); - return -ENODEV; + return PTR_ERR(i2c->dmach); } platform_set_drvdata(pdev, i2c); diff --git a/drivers/i2c/busses/i2c-nomadik.c b/drivers/i2c/busses/i2c-nomadik.c index 01a7d72e5511..e1e8d4ef9aa7 100644 --- a/drivers/i2c/busses/i2c-nomadik.c +++ b/drivers/i2c/busses/i2c-nomadik.c @@ -396,7 +396,7 @@ static void setup_i2c_controller(struct nmk_i2c_dev *dev) * 2 whereas it is 3 for fast and fastplus mode of * operation. TODO - high speed support. */ - div = (dev->clk_freq > 100000) ? 3 : 2; + div = (dev->clk_freq > I2C_MAX_STANDARD_MODE_FREQ) ? 3 : 2; /* * generate the mask for baud rate counters. The controller @@ -420,7 +420,7 @@ static void setup_i2c_controller(struct nmk_i2c_dev *dev) if (dev->sm > I2C_FREQ_MODE_FAST) { dev_err(&dev->adev->dev, "do not support this mode defaulting to std. mode\n"); - brcr2 = i2c_clk/(100000 * 2) & 0xffff; + brcr2 = i2c_clk / (I2C_MAX_STANDARD_MODE_FREQ * 2) & 0xffff; writel((brcr1 | brcr2), dev->virtbase + I2C_BRCR); writel(I2C_FREQ_MODE_STANDARD << 4, dev->virtbase + I2C_CR); @@ -949,10 +949,10 @@ static void nmk_i2c_of_probe(struct device_node *np, { /* Default to 100 kHz if no frequency is given in the node */ if (of_property_read_u32(np, "clock-frequency", &nmk->clk_freq)) - nmk->clk_freq = 100000; + nmk->clk_freq = I2C_MAX_STANDARD_MODE_FREQ; /* This driver only supports 'standard' and 'fast' modes of operation. */ - if (nmk->clk_freq <= 100000) + if (nmk->clk_freq <= I2C_MAX_STANDARD_MODE_FREQ) nmk->sm = I2C_FREQ_MODE_STANDARD; else nmk->sm = I2C_FREQ_MODE_FAST; diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c index 2dfea357b131..71b4637c86b7 100644 --- a/drivers/i2c/busses/i2c-omap.c +++ b/drivers/i2c/busses/i2c-omap.c @@ -1355,7 +1355,6 @@ omap_i2c_probe(struct platform_device *pdev) { struct omap_i2c_dev *omap; struct i2c_adapter *adap; - struct resource *mem; const struct omap_i2c_bus_platform_data *pdata = dev_get_platdata(&pdev->dev); struct device_node *node = pdev->dev.of_node; @@ -1375,14 +1374,13 @@ omap_i2c_probe(struct platform_device *pdev) if (!omap) return -ENOMEM; - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - omap->base = devm_ioremap_resource(&pdev->dev, mem); + omap->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(omap->base)) return PTR_ERR(omap->base); match = of_match_device(of_match_ptr(omap_i2c_of_match), &pdev->dev); if (match) { - u32 freq = 100000; /* default to 100000 Hz */ + u32 freq = I2C_MAX_STANDARD_MODE_FREQ; pdata = match->data; omap->flags = pdata->flags; diff --git a/drivers/i2c/busses/i2c-owl.c b/drivers/i2c/busses/i2c-owl.c index b6b5a495118b..3ab8be62c581 100644 --- a/drivers/i2c/busses/i2c-owl.c +++ b/drivers/i2c/busses/i2c-owl.c @@ -87,9 +87,6 @@ #define OWL_I2C_MAX_RETRIES 50 -#define OWL_I2C_DEF_SPEED_HZ 100000 -#define OWL_I2C_MAX_SPEED_HZ 400000 - struct owl_i2c_dev { struct i2c_adapter adap; struct i2c_msg *msg; @@ -419,11 +416,11 @@ static int owl_i2c_probe(struct platform_device *pdev) if (of_property_read_u32(dev->of_node, "clock-frequency", &i2c_dev->bus_freq)) - i2c_dev->bus_freq = OWL_I2C_DEF_SPEED_HZ; + i2c_dev->bus_freq = I2C_MAX_STANDARD_MODE_FREQ; /* We support only frequencies of 100k and 400k for now */ - if (i2c_dev->bus_freq != OWL_I2C_DEF_SPEED_HZ && - i2c_dev->bus_freq != OWL_I2C_MAX_SPEED_HZ) { + if (i2c_dev->bus_freq != I2C_MAX_STANDARD_MODE_FREQ && + i2c_dev->bus_freq != I2C_MAX_FAST_MODE_FREQ) { dev_err(dev, "invalid clock-frequency %d\n", i2c_dev->bus_freq); return -EINVAL; } diff --git a/drivers/i2c/busses/i2c-parport.c b/drivers/i2c/busses/i2c-parport.c index 81eb441b2387..a535889acca6 100644 --- a/drivers/i2c/busses/i2c-parport.c +++ b/drivers/i2c/busses/i2c-parport.c @@ -333,13 +333,17 @@ static void i2c_parport_attach(struct parport *port) /* Setup SMBus alert if supported */ if (adapter_parm[type].smbus_alert) { - adapter->ara = i2c_setup_smbus_alert(&adapter->adapter, - &adapter->alert_data); - if (adapter->ara) + struct i2c_client *ara; + + ara = i2c_new_smbus_alert_device(&adapter->adapter, + &adapter->alert_data); + if (!IS_ERR(ara)) { + adapter->ara = ara; parport_enable_irq(port); - else + } else { dev_warn(&adapter->pdev->dev, "Failed to register ARA client\n"); + } } /* Add the new adapter to the list */ diff --git a/drivers/i2c/busses/i2c-powermac.c b/drivers/i2c/busses/i2c-powermac.c index 973e5339033c..d565714c1f13 100644 --- a/drivers/i2c/busses/i2c-powermac.c +++ b/drivers/i2c/busses/i2c-powermac.c @@ -279,14 +279,13 @@ static bool i2c_powermac_get_type(struct i2c_adapter *adap, { char tmp[16]; - /* Note: we to _NOT_ want the standard - * i2c drivers to match with any of our powermac stuff - * unless they have been specifically modified to handle - * it on a case by case basis. For example, for thermal - * control, things like lm75 etc... shall match with their - * corresponding windfarm drivers, _NOT_ the generic ones, - * so we force a prefix of AAPL, onto the modalias to - * make that happen + /* + * Note: we do _NOT_ want the standard i2c drivers to match with any of + * our powermac stuff unless they have been specifically modified to + * handle it on a case by case basis. For example, for thermal control, + * things like lm75 etc... shall match with their corresponding + * windfarm drivers, _NOT_ the generic ones, so we force a prefix of + * 'MAC', onto the modalias to make that happen */ /* First try proper modalias */ diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index 17abf60c94ae..18d1e4fd4cf3 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -502,45 +501,40 @@ static int geni_i2c_probe(struct platform_device *pdev) struct resource *res; u32 proto, tx_depth; int ret; + struct device *dev = &pdev->dev; - gi2c = devm_kzalloc(&pdev->dev, sizeof(*gi2c), GFP_KERNEL); + gi2c = devm_kzalloc(dev, sizeof(*gi2c), GFP_KERNEL); if (!gi2c) return -ENOMEM; - gi2c->se.dev = &pdev->dev; - gi2c->se.wrapper = dev_get_drvdata(pdev->dev.parent); + gi2c->se.dev = dev; + gi2c->se.wrapper = dev_get_drvdata(dev->parent); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - gi2c->se.base = devm_ioremap_resource(&pdev->dev, res); + gi2c->se.base = devm_ioremap_resource(dev, res); if (IS_ERR(gi2c->se.base)) return PTR_ERR(gi2c->se.base); - gi2c->se.clk = devm_clk_get(&pdev->dev, "se"); - if (IS_ERR(gi2c->se.clk) && !has_acpi_companion(&pdev->dev)) { - ret = PTR_ERR(gi2c->se.clk); - dev_err(&pdev->dev, "Err getting SE Core clk %d\n", ret); - return ret; - } + gi2c->se.clk = devm_clk_get(dev, "se"); + if (IS_ERR(gi2c->se.clk) && !has_acpi_companion(dev)) + return PTR_ERR(gi2c->se.clk); - ret = device_property_read_u32(&pdev->dev, "clock-frequency", - &gi2c->clk_freq_out); + ret = device_property_read_u32(dev, "clock-frequency", + &gi2c->clk_freq_out); if (ret) { - dev_info(&pdev->dev, - "Bus frequency not specified, default to 100kHz.\n"); + dev_info(dev, "Bus frequency not specified, default to 100kHz.\n"); gi2c->clk_freq_out = KHZ(100); } - if (has_acpi_companion(&pdev->dev)) - ACPI_COMPANION_SET(&gi2c->adap.dev, ACPI_COMPANION(&pdev->dev)); + if (has_acpi_companion(dev)) + ACPI_COMPANION_SET(&gi2c->adap.dev, ACPI_COMPANION(dev)); gi2c->irq = platform_get_irq(pdev, 0); - if (gi2c->irq < 0) { - dev_err(&pdev->dev, "IRQ error for i2c-geni\n"); + if (gi2c->irq < 0) return gi2c->irq; - } ret = geni_i2c_clk_map_idx(gi2c); if (ret) { - dev_err(&pdev->dev, "Invalid clk frequency %d Hz: %d\n", + dev_err(dev, "Invalid clk frequency %d Hz: %d\n", gi2c->clk_freq_out, ret); return ret; } @@ -549,29 +543,29 @@ static int geni_i2c_probe(struct platform_device *pdev) init_completion(&gi2c->done); spin_lock_init(&gi2c->lock); platform_set_drvdata(pdev, gi2c); - ret = devm_request_irq(&pdev->dev, gi2c->irq, geni_i2c_irq, - IRQF_TRIGGER_HIGH, "i2c_geni", gi2c); + ret = devm_request_irq(dev, gi2c->irq, geni_i2c_irq, 0, + dev_name(dev), gi2c); if (ret) { - dev_err(&pdev->dev, "Request_irq failed:%d: err:%d\n", + dev_err(dev, "Request_irq failed:%d: err:%d\n", gi2c->irq, ret); return ret; } /* Disable the interrupt so that the system can enter low-power mode */ disable_irq(gi2c->irq); i2c_set_adapdata(&gi2c->adap, gi2c); - gi2c->adap.dev.parent = &pdev->dev; - gi2c->adap.dev.of_node = pdev->dev.of_node; + gi2c->adap.dev.parent = dev; + gi2c->adap.dev.of_node = dev->of_node; strlcpy(gi2c->adap.name, "Geni-I2C", sizeof(gi2c->adap.name)); ret = geni_se_resources_on(&gi2c->se); if (ret) { - dev_err(&pdev->dev, "Error turning on resources %d\n", ret); + dev_err(dev, "Error turning on resources %d\n", ret); return ret; } proto = geni_se_read_proto(&gi2c->se); tx_depth = geni_se_get_tx_fifo_depth(&gi2c->se); if (proto != GENI_SE_I2C) { - dev_err(&pdev->dev, "Invalid proto %d\n", proto); + dev_err(dev, "Invalid proto %d\n", proto); geni_se_resources_off(&gi2c->se); return -ENXIO; } @@ -581,11 +575,11 @@ static int geni_i2c_probe(struct platform_device *pdev) true, true, true); ret = geni_se_resources_off(&gi2c->se); if (ret) { - dev_err(&pdev->dev, "Error turning off resources %d\n", ret); + dev_err(dev, "Error turning off resources %d\n", ret); return ret; } - dev_dbg(&pdev->dev, "i2c fifo/se-dma mode. fifo depth:%d\n", tx_depth); + dev_dbg(dev, "i2c fifo/se-dma mode. fifo depth:%d\n", tx_depth); gi2c->suspended = 1; pm_runtime_set_suspended(gi2c->se.dev); @@ -595,12 +589,12 @@ static int geni_i2c_probe(struct platform_device *pdev) ret = i2c_add_adapter(&gi2c->adap); if (ret) { - dev_err(&pdev->dev, "Error adding i2c adapter %d\n", ret); + dev_err(dev, "Error adding i2c adapter %d\n", ret); pm_runtime_disable(gi2c->se.dev); return ret; } - dev_dbg(&pdev->dev, "Geni-I2C adaptor successfully added\n"); + dev_dbg(dev, "Geni-I2C adaptor successfully added\n"); return 0; } diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c index 2d7dabe12723..748872a9b0fc 100644 --- a/drivers/i2c/busses/i2c-qup.c +++ b/drivers/i2c/busses/i2c-qup.c @@ -136,13 +136,8 @@ */ #define TOUT_MIN 2 -/* I2C Frequency Modes */ -#define I2C_STANDARD_FREQ 100000 -#define I2C_FAST_MODE_FREQ 400000 -#define I2C_FAST_MODE_PLUS_FREQ 1000000 - /* Default values. Use these if FW query fails */ -#define DEFAULT_CLK_FREQ I2C_STANDARD_FREQ +#define DEFAULT_CLK_FREQ I2C_MAX_STANDARD_MODE_FREQ #define DEFAULT_SRC_CLK 20000000 /* @@ -1756,7 +1751,7 @@ static int qup_i2c_probe(struct platform_device *pdev) nodma: /* We support frequencies up to FAST Mode Plus (1MHz) */ - if (!clk_freq || clk_freq > I2C_FAST_MODE_PLUS_FREQ) { + if (!clk_freq || clk_freq > I2C_MAX_FAST_MODE_PLUS_FREQ) { dev_err(qup->dev, "clock frequency not supported %d\n", clk_freq); return -EINVAL; @@ -1861,7 +1856,7 @@ static int qup_i2c_probe(struct platform_device *pdev) qup->in_fifo_sz = qup->in_blk_sz * (2 << size); hs_div = 3; - if (clk_freq <= I2C_STANDARD_FREQ) { + if (clk_freq <= I2C_MAX_STANDARD_MODE_FREQ) { fs_div = ((src_clk_freq / clk_freq) / 2) - 3; qup->clk_ctl = (hs_div << 8) | (fs_div & 0xff); } else { diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c index 879f0e61a496..3b5397aa4ca6 100644 --- a/drivers/i2c/busses/i2c-rcar.c +++ b/drivers/i2c/busses/i2c-rcar.c @@ -235,17 +235,20 @@ static int rcar_i2c_bus_barrier(struct rcar_i2c_priv *priv) return i2c_recover_bus(&priv->adap); } -static int rcar_i2c_clock_calculate(struct rcar_i2c_priv *priv, struct i2c_timings *t) +static int rcar_i2c_clock_calculate(struct rcar_i2c_priv *priv) { u32 scgd, cdf, round, ick, sum, scl, cdf_width; unsigned long rate; struct device *dev = rcar_i2c_priv_to_dev(priv); + struct i2c_timings t = { + .bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ, + .scl_fall_ns = 35, + .scl_rise_ns = 200, + .scl_int_delay_ns = 50, + }; /* Fall back to previously used values if not supplied */ - t->bus_freq_hz = t->bus_freq_hz ?: 100000; - t->scl_fall_ns = t->scl_fall_ns ?: 35; - t->scl_rise_ns = t->scl_rise_ns ?: 200; - t->scl_int_delay_ns = t->scl_int_delay_ns ?: 50; + i2c_parse_fw_timings(dev, &t, false); switch (priv->devtype) { case I2C_RCAR_GEN1: @@ -291,7 +294,7 @@ static int rcar_i2c_clock_calculate(struct rcar_i2c_priv *priv, struct i2c_timin * = F[sum * ick / 1000000000] * = F[(ick / 1000000) * sum / 1000] */ - sum = t->scl_fall_ns + t->scl_rise_ns + t->scl_int_delay_ns; + sum = t.scl_fall_ns + t.scl_rise_ns + t.scl_int_delay_ns; round = (ick + 500000) / 1000000 * sum; round = (round + 500) / 1000; @@ -309,7 +312,7 @@ static int rcar_i2c_clock_calculate(struct rcar_i2c_priv *priv, struct i2c_timin */ for (scgd = 0; scgd < 0x40; scgd++) { scl = ick / (20 + (scgd * 8) + round); - if (scl <= t->bus_freq_hz) + if (scl <= t.bus_freq_hz) goto scgd_find; } dev_err(dev, "it is impossible to calculate best SCL\n"); @@ -317,7 +320,7 @@ static int rcar_i2c_clock_calculate(struct rcar_i2c_priv *priv, struct i2c_timin scgd_find: dev_dbg(dev, "clk %d/%d(%lu), round %u, CDF:0x%x, SCGD: 0x%x\n", - scl, t->bus_freq_hz, rate, round, cdf, scgd); + scl, t.bus_freq_hz, rate, round, cdf, scgd); /* keep icccr value */ priv->icccr = scgd << cdf_width | cdf; @@ -920,7 +923,6 @@ static int rcar_i2c_probe(struct platform_device *pdev) struct rcar_i2c_priv *priv; struct i2c_adapter *adap; struct device *dev = &pdev->dev; - struct i2c_timings i2c_t; int ret; /* Otherwise logic will break because some bytes must always use PIO */ @@ -957,8 +959,6 @@ static int rcar_i2c_probe(struct platform_device *pdev) i2c_set_adapdata(adap, priv); strlcpy(adap->name, pdev->name, sizeof(adap->name)); - i2c_parse_fw_timings(dev, &i2c_t, false); - /* Init DMA */ sg_init_table(&priv->sg, 1); priv->dma_direction = DMA_NONE; @@ -967,7 +967,7 @@ static int rcar_i2c_probe(struct platform_device *pdev) /* Activate device for clock calculation */ pm_runtime_enable(dev); pm_runtime_get_sync(dev); - ret = rcar_i2c_clock_calculate(priv, &i2c_t); + ret = rcar_i2c_clock_calculate(priv); if (ret < 0) goto out_pm_put; diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c index 800414886f6b..4eccc0f69861 100644 --- a/drivers/i2c/busses/i2c-riic.c +++ b/drivers/i2c/busses/i2c-riic.c @@ -287,10 +287,10 @@ static int riic_init_hw(struct riic_dev *riic, struct i2c_timings *t) pm_runtime_get_sync(riic->adapter.dev.parent); - if (t->bus_freq_hz > 400000) { + if (t->bus_freq_hz > I2C_MAX_FAST_MODE_FREQ) { dev_err(&riic->adapter.dev, - "unsupported bus speed (%dHz). 400000 max\n", - t->bus_freq_hz); + "unsupported bus speed (%dHz). %d max\n", + t->bus_freq_hz, I2C_MAX_FAST_MODE_FREQ); ret = -EINVAL; goto out; } diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c index 1a33007b03e9..73272d4296bb 100644 --- a/drivers/i2c/busses/i2c-rk3x.c +++ b/drivers/i2c/busses/i2c-rk3x.c @@ -539,9 +539,9 @@ static irqreturn_t rk3x_i2c_irq(int irqno, void *dev_id) */ static const struct i2c_spec_values *rk3x_i2c_get_spec(unsigned int speed) { - if (speed <= 100000) + if (speed <= I2C_MAX_STANDARD_MODE_FREQ) return &standard_mode_spec; - else if (speed <= 400000) + else if (speed <= I2C_MAX_FAST_MODE_FREQ) return &fast_mode_spec; else return &fast_mode_plus_spec; @@ -578,8 +578,8 @@ static int rk3x_i2c_v0_calc_timings(unsigned long clk_rate, int ret = 0; /* Only support standard-mode and fast-mode */ - if (WARN_ON(t->bus_freq_hz > 400000)) - t->bus_freq_hz = 400000; + if (WARN_ON(t->bus_freq_hz > I2C_MAX_FAST_MODE_FREQ)) + t->bus_freq_hz = I2C_MAX_FAST_MODE_FREQ; /* prevent scl_rate_khz from becoming 0 */ if (WARN_ON(t->bus_freq_hz < 1000)) @@ -758,8 +758,8 @@ static int rk3x_i2c_v1_calc_timings(unsigned long clk_rate, int ret = 0; /* Support standard-mode, fast-mode and fast-mode plus */ - if (WARN_ON(t->bus_freq_hz > 1000000)) - t->bus_freq_hz = 1000000; + if (WARN_ON(t->bus_freq_hz > I2C_MAX_FAST_MODE_PLUS_FREQ)) + t->bus_freq_hz = I2C_MAX_FAST_MODE_PLUS_FREQ; /* prevent scl_rate_khz from becoming 0 */ if (WARN_ON(t->bus_freq_hz < 1000)) diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c index c98ef4c4a0c9..5a5638e1daa1 100644 --- a/drivers/i2c/busses/i2c-s3c2410.c +++ b/drivers/i2c/busses/i2c-s3c2410.c @@ -835,11 +835,11 @@ static int s3c24xx_i2c_clockrate(struct s3c24xx_i2c *i2c, unsigned int *got) int freq; i2c->clkrate = clkin; - clkin /= 1000; /* clkin now in KHz */ + clkin /= 1000; /* clkin now in KHz */ dev_dbg(i2c->dev, "pdata desired frequency %lu\n", pdata->frequency); - target_frequency = pdata->frequency ? pdata->frequency : 100000; + target_frequency = pdata->frequency ?: I2C_MAX_STANDARD_MODE_FREQ; target_frequency /= 1000; /* Target frequency now in KHz */ diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c index 82b3b795e0bd..d83ca4028fa0 100644 --- a/drivers/i2c/busses/i2c-sh_mobile.c +++ b/drivers/i2c/busses/i2c-sh_mobile.c @@ -145,9 +145,6 @@ struct sh_mobile_dt_config { #define IIC_FLAG_HAS_ICIC67 (1 << 0) -#define STANDARD_MODE 100000 -#define FAST_MODE 400000 - /* Register offsets */ #define ICDR 0x00 #define ICCR 0x04 @@ -270,11 +267,11 @@ static int sh_mobile_i2c_init(struct sh_mobile_i2c_data *pd) i2c_clk_khz = clk_get_rate(pd->clk) / 1000 / pd->clks_per_count; - if (pd->bus_speed == STANDARD_MODE) { + if (pd->bus_speed == I2C_MAX_STANDARD_MODE_FREQ) { tLOW = 47; /* tLOW = 4.7 us */ tHIGH = 40; /* tHD;STA = tHIGH = 4.0 us */ tf = 3; /* tf = 0.3 us */ - } else if (pd->bus_speed == FAST_MODE) { + } else if (pd->bus_speed == I2C_MAX_FAST_MODE_FREQ) { tLOW = 13; /* tLOW = 1.3 us */ tHIGH = 6; /* tHD;STA = tHIGH = 0.6 us */ tf = 3; /* tf = 0.3 us */ @@ -851,7 +848,7 @@ static int sh_mobile_i2c_probe(struct platform_device *dev) return PTR_ERR(pd->reg); ret = of_property_read_u32(dev->dev.of_node, "clock-frequency", &bus_speed); - pd->bus_speed = (ret || !bus_speed) ? STANDARD_MODE : bus_speed; + pd->bus_speed = (ret || !bus_speed) ? I2C_MAX_STANDARD_MODE_FREQ : bus_speed; pd->clks_per_count = 1; /* Newer variants come with two new bits in ICIC */ diff --git a/drivers/i2c/busses/i2c-sirf.c b/drivers/i2c/busses/i2c-sirf.c index fb7a046b3226..a459e00c6851 100644 --- a/drivers/i2c/busses/i2c-sirf.c +++ b/drivers/i2c/busses/i2c-sirf.c @@ -62,7 +62,6 @@ #define SIRFSOC_I2C_STOP BIT(6) #define SIRFSOC_I2C_START BIT(7) -#define SIRFSOC_I2C_DEFAULT_SPEED 100000 #define SIRFSOC_I2C_ERR_NOACK 1 #define SIRFSOC_I2C_ERR_TIMEOUT 2 @@ -353,7 +352,7 @@ static int i2c_sirfsoc_probe(struct platform_device *pdev) err = of_property_read_u32(pdev->dev.of_node, "clock-frequency", &bitrate); if (err < 0) - bitrate = SIRFSOC_I2C_DEFAULT_SPEED; + bitrate = I2C_MAX_STANDARD_MODE_FREQ; /* * Due to some hardware design issues, we need to tune the formula. diff --git a/drivers/i2c/busses/i2c-sprd.c b/drivers/i2c/busses/i2c-sprd.c index b432e7580458..123a42bfe3b1 100644 --- a/drivers/i2c/busses/i2c-sprd.c +++ b/drivers/i2c/busses/i2c-sprd.c @@ -337,9 +337,9 @@ static void sprd_i2c_set_clk(struct sprd_i2c *i2c_dev, u32 freq) writel(div1, i2c_dev->base + ADDR_DVD1); /* Start hold timing = hold time(us) * source clock */ - if (freq == 400000) + if (freq == I2C_MAX_FAST_MODE_FREQ) writel((6 * apb_clk) / 10000000, i2c_dev->base + ADDR_STA0_DVD); - else if (freq == 100000) + else if (freq == I2C_MAX_STANDARD_MODE_FREQ) writel((4 * apb_clk) / 1000000, i2c_dev->base + ADDR_STA0_DVD); } @@ -502,7 +502,7 @@ static int sprd_i2c_probe(struct platform_device *pdev) snprintf(i2c_dev->adap.name, sizeof(i2c_dev->adap.name), "%s", "sprd-i2c"); - i2c_dev->bus_freq = 100000; + i2c_dev->bus_freq = I2C_MAX_STANDARD_MODE_FREQ; i2c_dev->adap.owner = THIS_MODULE; i2c_dev->dev = dev; i2c_dev->adap.retries = 3; @@ -516,7 +516,8 @@ static int sprd_i2c_probe(struct platform_device *pdev) i2c_dev->bus_freq = prop; /* We only support 100k and 400k now, otherwise will return error. */ - if (i2c_dev->bus_freq != 100000 && i2c_dev->bus_freq != 400000) + if (i2c_dev->bus_freq != I2C_MAX_STANDARD_MODE_FREQ && + i2c_dev->bus_freq != I2C_MAX_FAST_MODE_FREQ) return -EINVAL; ret = sprd_i2c_clk_init(i2c_dev); diff --git a/drivers/i2c/busses/i2c-st.c b/drivers/i2c/busses/i2c-st.c index f7f7b5b64720..faa81a95551f 100644 --- a/drivers/i2c/busses/i2c-st.c +++ b/drivers/i2c/busses/i2c-st.c @@ -213,7 +213,7 @@ static inline void st_i2c_clr_bits(void __iomem *reg, u32 mask) */ static struct st_i2c_timings i2c_timings[] = { [I2C_MODE_STANDARD] = { - .rate = 100000, + .rate = I2C_MAX_STANDARD_MODE_FREQ, .rep_start_hold = 4400, .rep_start_setup = 5170, .start_hold = 4400, @@ -222,7 +222,7 @@ static struct st_i2c_timings i2c_timings[] = { .bus_free_time = 5170, }, [I2C_MODE_FAST] = { - .rate = 400000, + .rate = I2C_MAX_FAST_MODE_FREQ, .rep_start_hold = 660, .rep_start_setup = 660, .start_hold = 660, @@ -836,7 +836,7 @@ static int st_i2c_probe(struct platform_device *pdev) i2c_dev->mode = I2C_MODE_STANDARD; ret = of_property_read_u32(np, "clock-frequency", &clk_rate); - if ((!ret) && (clk_rate == 400000)) + if (!ret && (clk_rate == I2C_MAX_FAST_MODE_FREQ)) i2c_dev->mode = I2C_MODE_FAST; i2c_dev->dev = &pdev->dev; diff --git a/drivers/i2c/busses/i2c-stm32f4.c b/drivers/i2c/busses/i2c-stm32f4.c index ba600d77a3f8..d6a69dfcac3f 100644 --- a/drivers/i2c/busses/i2c-stm32f4.c +++ b/drivers/i2c/busses/i2c-stm32f4.c @@ -232,10 +232,10 @@ static void stm32f4_i2c_set_speed_mode(struct stm32f4_i2c_dev *i2c_dev) * In standard mode: * t_scl_high = t_scl_low = CCR * I2C parent clk period * So to reach 100 kHz, we have: - * CCR = I2C parent rate / 100 kHz >> 1 + * CCR = I2C parent rate / (100 kHz * 2) * * For example with parent rate = 2 MHz: - * CCR = 2000000 / (100000 << 1) = 10 + * CCR = 2000000 / (100000 * 2) = 10 * t_scl_high = t_scl_low = 10 * (1 / 2000000) = 5000 ns * t_scl_high + t_scl_low = 10000 ns so 100 kHz is reached * @@ -243,7 +243,7 @@ static void stm32f4_i2c_set_speed_mode(struct stm32f4_i2c_dev *i2c_dev) * parent rate is not higher than 46 MHz . As a result val * is at most 8 bits wide and so fits into the CCR bits [11:0]. */ - val = i2c_dev->parent_rate / (100000 << 1); + val = i2c_dev->parent_rate / (I2C_MAX_STANDARD_MODE_FREQ * 2); } else { /* * In fast mode, we compute CCR with duty = 0 as with low @@ -263,7 +263,7 @@ static void stm32f4_i2c_set_speed_mode(struct stm32f4_i2c_dev *i2c_dev) * parent rate is not higher than 46 MHz . As a result val * is at most 6 bits wide and so fits into the CCR bits [11:0]. */ - val = DIV_ROUND_UP(i2c_dev->parent_rate, 400000 * 3); + val = DIV_ROUND_UP(i2c_dev->parent_rate, I2C_MAX_FAST_MODE_FREQ * 3); /* Select Fast mode */ ccr |= STM32F4_I2C_CCR_FS; @@ -807,7 +807,7 @@ static int stm32f4_i2c_probe(struct platform_device *pdev) i2c_dev->speed = STM32_I2C_SPEED_STANDARD; ret = of_property_read_u32(np, "clock-frequency", &clk_rate); - if (!ret && clk_rate >= 400000) + if (!ret && clk_rate >= I2C_MAX_FAST_MODE_FREQ) i2c_dev->speed = STM32_I2C_SPEED_FAST; i2c_dev->dev = &pdev->dev; diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c index 5c3e8ac6ad92..330ffed011e0 100644 --- a/drivers/i2c/busses/i2c-stm32f7.c +++ b/drivers/i2c/busses/i2c-stm32f7.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,7 @@ /* STM32F7 I2C control 1 */ #define STM32F7_I2C_CR1_PECEN BIT(23) +#define STM32F7_I2C_CR1_WUPEN BIT(18) #define STM32F7_I2C_CR1_SBC BIT(16) #define STM32F7_I2C_CR1_RXDMAEN BIT(15) #define STM32F7_I2C_CR1_TXDMAEN BIT(14) @@ -174,7 +176,6 @@ * @cr2: Control register 2 * @oar1: Own address 1 register * @oar2: Own address 2 register - * @pecr: PEC register * @tmgr: Timing register */ struct stm32f7_i2c_regs { @@ -182,7 +183,6 @@ struct stm32f7_i2c_regs { u32 cr2; u32 oar1; u32 oar2; - u32 pecr; u32 tmgr; }; @@ -221,6 +221,7 @@ struct stm32f7_i2c_spec { * @fall_time: Fall time (ns) * @dnf: Digital filter coefficient (0-16) * @analog_filter: Analog filter delay (On/Off) + * @fmp_clr_offset: Fast Mode Plus clear register offset from set register */ struct stm32f7_i2c_setup { enum stm32_i2c_speed speed; @@ -230,6 +231,7 @@ struct stm32f7_i2c_setup { u32 fall_time; u8 dnf; bool analog_filter; + u32 fmp_clr_offset; }; /** @@ -301,6 +303,10 @@ struct stm32f7_i2c_msg { * @dma: dma data * @use_dma: boolean to know if dma is used in the current transfer * @regmap: holds SYSCFG phandle for Fast Mode Plus bits + * @fmp_sreg: register address for setting Fast Mode Plus bits + * @fmp_creg: register address for clearing Fast Mode Plus bits + * @fmp_mask: mask for Fast Mode Plus bits in set register + * @wakeup_src: boolean to know if the device is a wakeup source */ struct stm32f7_i2c_dev { struct i2c_adapter adap; @@ -323,6 +329,10 @@ struct stm32f7_i2c_dev { struct stm32_i2c_dma *dma; bool use_dma; struct regmap *regmap; + u32 fmp_sreg; + u32 fmp_creg; + u32 fmp_mask; + bool wakeup_src; }; /* @@ -334,9 +344,9 @@ struct stm32f7_i2c_dev { */ static struct stm32f7_i2c_spec i2c_specs[] = { [STM32_I2C_SPEED_STANDARD] = { - .rate = 100000, - .rate_min = 80000, - .rate_max = 100000, + .rate = I2C_MAX_STANDARD_MODE_FREQ, + .rate_min = I2C_MAX_STANDARD_MODE_FREQ * 8 / 10, /* 80% */ + .rate_max = I2C_MAX_STANDARD_MODE_FREQ, .fall_max = 300, .rise_max = 1000, .hddat_min = 0, @@ -346,9 +356,9 @@ static struct stm32f7_i2c_spec i2c_specs[] = { .h_min = 4000, }, [STM32_I2C_SPEED_FAST] = { - .rate = 400000, - .rate_min = 320000, - .rate_max = 400000, + .rate = I2C_MAX_FAST_MODE_FREQ, + .rate_min = I2C_MAX_FAST_MODE_FREQ * 8 / 10, /* 80% */ + .rate_max = I2C_MAX_FAST_MODE_FREQ, .fall_max = 300, .rise_max = 300, .hddat_min = 0, @@ -358,9 +368,9 @@ static struct stm32f7_i2c_spec i2c_specs[] = { .h_min = 600, }, [STM32_I2C_SPEED_FAST_PLUS] = { - .rate = 1000000, - .rate_min = 800000, - .rate_max = 1000000, + .rate = I2C_MAX_FAST_MODE_PLUS_FREQ, + .rate_min = I2C_MAX_FAST_MODE_PLUS_FREQ * 8 / 10, /* 80% */ + .rate_max = I2C_MAX_FAST_MODE_PLUS_FREQ, .fall_max = 100, .rise_max = 120, .hddat_min = 0, @@ -378,6 +388,14 @@ static const struct stm32f7_i2c_setup stm32f7_setup = { .analog_filter = STM32F7_I2C_ANALOG_FILTER_ENABLE, }; +static const struct stm32f7_i2c_setup stm32mp15_setup = { + .rise_time = STM32F7_I2C_RISE_TIME_DEFAULT, + .fall_time = STM32F7_I2C_FALL_TIME_DEFAULT, + .dnf = STM32F7_I2C_DNF_DEFAULT, + .analog_filter = STM32F7_I2C_ANALOG_FILTER_ENABLE, + .fmp_clr_offset = 0x40, +}; + static inline void stm32f7_i2c_set_bits(void __iomem *reg, u32 mask) { writel_relaxed(readl_relaxed(reg) | mask, reg); @@ -592,8 +610,25 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev, struct stm32f7_i2c_setup *setup) { + struct i2c_timings timings, *t = &timings; int ret = 0; + t->bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; + t->scl_rise_ns = i2c_dev->setup.rise_time; + t->scl_fall_ns = i2c_dev->setup.fall_time; + + i2c_parse_fw_timings(i2c_dev->dev, t, false); + + if (t->bus_freq_hz >= I2C_MAX_FAST_MODE_PLUS_FREQ) + i2c_dev->speed = STM32_I2C_SPEED_FAST_PLUS; + else if (t->bus_freq_hz >= I2C_MAX_FAST_MODE_FREQ) + i2c_dev->speed = STM32_I2C_SPEED_FAST; + else + i2c_dev->speed = STM32_I2C_SPEED_STANDARD; + + i2c_dev->setup.rise_time = t->scl_rise_ns; + i2c_dev->setup.fall_time = t->scl_fall_ns; + setup->speed = i2c_dev->speed; setup->speed_freq = i2c_specs[setup->speed].rate; setup->clock_src = clk_get_rate(i2c_dev->clk); @@ -1691,6 +1726,24 @@ static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr, return ret; } +static void stm32f7_i2c_enable_wakeup(struct stm32f7_i2c_dev *i2c_dev, + bool enable) +{ + void __iomem *base = i2c_dev->base; + u32 mask = STM32F7_I2C_CR1_WUPEN; + + if (!i2c_dev->wakeup_src) + return; + + if (enable) { + device_set_wakeup_enable(i2c_dev->dev, true); + stm32f7_i2c_set_bits(base + STM32F7_I2C_CR1, mask); + } else { + device_set_wakeup_enable(i2c_dev->dev, false); + stm32f7_i2c_clr_bits(base + STM32F7_I2C_CR1, mask); + } +} + static int stm32f7_i2c_reg_slave(struct i2c_client *slave) { struct stm32f7_i2c_dev *i2c_dev = i2c_get_adapdata(slave->adapter); @@ -1717,6 +1770,9 @@ static int stm32f7_i2c_reg_slave(struct i2c_client *slave) if (ret < 0) return ret; + if (!stm32f7_i2c_is_slave_registered(i2c_dev)) + stm32f7_i2c_enable_wakeup(i2c_dev, true); + if (id == 0) { /* Configure Own Address 1 */ oar1 = readl_relaxed(i2c_dev->base + STM32F7_I2C_OAR1); @@ -1758,6 +1814,9 @@ static int stm32f7_i2c_reg_slave(struct i2c_client *slave) ret = 0; pm_free: + if (!stm32f7_i2c_is_slave_registered(i2c_dev)) + stm32f7_i2c_enable_wakeup(i2c_dev, false); + pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); @@ -1791,8 +1850,10 @@ static int stm32f7_i2c_unreg_slave(struct i2c_client *slave) i2c_dev->slave[id] = NULL; - if (!(stm32f7_i2c_is_slave_registered(i2c_dev))) + if (!stm32f7_i2c_is_slave_registered(i2c_dev)) { stm32f7_i2c_disable_irq(i2c_dev, STM32F7_I2C_ALL_IRQ_MASK); + stm32f7_i2c_enable_wakeup(i2c_dev, false); + } pm_runtime_mark_last_busy(i2c_dev->dev); pm_runtime_put_autosuspend(i2c_dev->dev); @@ -1800,28 +1861,51 @@ static int stm32f7_i2c_unreg_slave(struct i2c_client *slave) return 0; } +static int stm32f7_i2c_write_fm_plus_bits(struct stm32f7_i2c_dev *i2c_dev, + bool enable) +{ + int ret; + + if (i2c_dev->speed != STM32_I2C_SPEED_FAST_PLUS || + IS_ERR_OR_NULL(i2c_dev->regmap)) + /* Optional */ + return 0; + + if (i2c_dev->fmp_sreg == i2c_dev->fmp_creg) + ret = regmap_update_bits(i2c_dev->regmap, + i2c_dev->fmp_sreg, + i2c_dev->fmp_mask, + enable ? i2c_dev->fmp_mask : 0); + else + ret = regmap_write(i2c_dev->regmap, + enable ? i2c_dev->fmp_sreg : + i2c_dev->fmp_creg, + i2c_dev->fmp_mask); + + return ret; +} + static int stm32f7_i2c_setup_fm_plus_bits(struct platform_device *pdev, struct stm32f7_i2c_dev *i2c_dev) { struct device_node *np = pdev->dev.of_node; int ret; - u32 reg, mask; i2c_dev->regmap = syscon_regmap_lookup_by_phandle(np, "st,syscfg-fmp"); - if (IS_ERR(i2c_dev->regmap)) { + if (IS_ERR(i2c_dev->regmap)) /* Optional */ return 0; - } - ret = of_property_read_u32_index(np, "st,syscfg-fmp", 1, ®); + ret = of_property_read_u32_index(np, "st,syscfg-fmp", 1, + &i2c_dev->fmp_sreg); if (ret) return ret; - ret = of_property_read_u32_index(np, "st,syscfg-fmp", 2, &mask); - if (ret) - return ret; + i2c_dev->fmp_creg = i2c_dev->fmp_sreg + + i2c_dev->setup.fmp_clr_offset; - return regmap_update_bits(i2c_dev->regmap, reg, mask, mask); + return of_property_read_u32_index(np, "st,syscfg-fmp", 2, + &i2c_dev->fmp_mask); } static u32 stm32f7_i2c_func(struct i2c_adapter *adap) @@ -1847,7 +1931,6 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) struct stm32f7_i2c_dev *i2c_dev; const struct stm32f7_i2c_setup *setup; struct resource *res; - u32 clk_rate, rise_time, fall_time; struct i2c_adapter *adap; struct reset_control *rst; dma_addr_t phy_addr; @@ -1879,6 +1962,9 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) return irq_error ? : -ENOENT; } + i2c_dev->wakeup_src = of_property_read_bool(pdev->dev.of_node, + "wakeup-source"); + i2c_dev->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(i2c_dev->clk)) { dev_err(&pdev->dev, "Error: Missing controller clock\n"); @@ -1891,20 +1977,6 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) return ret; } - i2c_dev->speed = STM32_I2C_SPEED_STANDARD; - ret = device_property_read_u32(&pdev->dev, "clock-frequency", - &clk_rate); - if (!ret && clk_rate >= 1000000) { - i2c_dev->speed = STM32_I2C_SPEED_FAST_PLUS; - ret = stm32f7_i2c_setup_fm_plus_bits(pdev, i2c_dev); - if (ret) - goto clk_free; - } else if (!ret && clk_rate >= 400000) { - i2c_dev->speed = STM32_I2C_SPEED_FAST; - } else if (!ret && clk_rate >= 100000) { - i2c_dev->speed = STM32_I2C_SPEED_STANDARD; - } - rst = devm_reset_control_get(&pdev->dev, NULL); if (IS_ERR(rst)) { dev_err(&pdev->dev, "Error: Missing controller reset\n"); @@ -1944,20 +2016,19 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) } i2c_dev->setup = *setup; - ret = device_property_read_u32(i2c_dev->dev, "i2c-scl-rising-time-ns", - &rise_time); - if (!ret) - i2c_dev->setup.rise_time = rise_time; - - ret = device_property_read_u32(i2c_dev->dev, "i2c-scl-falling-time-ns", - &fall_time); - if (!ret) - i2c_dev->setup.fall_time = fall_time; - ret = stm32f7_i2c_setup_timing(i2c_dev, &i2c_dev->setup); if (ret) goto clk_free; + if (i2c_dev->speed == STM32_I2C_SPEED_FAST_PLUS) { + ret = stm32f7_i2c_setup_fm_plus_bits(pdev, i2c_dev); + if (ret) + goto clk_free; + ret = stm32f7_i2c_write_fm_plus_bits(i2c_dev, true); + if (ret) + goto clk_free; + } + adap = &i2c_dev->adap; i2c_set_adapdata(adap, i2c_dev); snprintf(adap->name, sizeof(adap->name), "STM32F7 I2C(%pa)", @@ -1982,7 +2053,17 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) if (ret != -EPROBE_DEFER) dev_err(&pdev->dev, "Failed to request dma error %i\n", ret); - goto clk_free; + goto fmp_clear; + } + + if (i2c_dev->wakeup_src) { + device_set_wakeup_capable(i2c_dev->dev, true); + + ret = dev_pm_set_wake_irq(i2c_dev->dev, irq_event); + if (ret) { + dev_err(i2c_dev->dev, "Failed to set wake up irq\n"); + goto clr_wakeup_capable; + } } platform_set_drvdata(pdev, i2c_dev); @@ -2014,11 +2095,21 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) pm_runtime_set_suspended(i2c_dev->dev); pm_runtime_dont_use_autosuspend(i2c_dev->dev); + if (i2c_dev->wakeup_src) + dev_pm_clear_wake_irq(i2c_dev->dev); + +clr_wakeup_capable: + if (i2c_dev->wakeup_src) + device_set_wakeup_capable(i2c_dev->dev, false); + if (i2c_dev->dma) { stm32_i2c_dma_free(i2c_dev->dma); i2c_dev->dma = NULL; } +fmp_clear: + stm32f7_i2c_write_fm_plus_bits(i2c_dev, false); + clk_free: clk_disable_unprepare(i2c_dev->clk); @@ -2032,6 +2123,15 @@ static int stm32f7_i2c_remove(struct platform_device *pdev) i2c_del_adapter(&i2c_dev->adap); pm_runtime_get_sync(i2c_dev->dev); + if (i2c_dev->wakeup_src) { + dev_pm_clear_wake_irq(i2c_dev->dev); + /* + * enforce that wakeup is disabled and that the device + * is marked as non wakeup capable + */ + device_init_wakeup(i2c_dev->dev, false); + } + pm_runtime_put_noidle(i2c_dev->dev); pm_runtime_disable(i2c_dev->dev); pm_runtime_set_suspended(i2c_dev->dev); @@ -2042,6 +2142,8 @@ static int stm32f7_i2c_remove(struct platform_device *pdev) i2c_dev->dma = NULL; } + stm32f7_i2c_write_fm_plus_bits(i2c_dev, false); + clk_disable_unprepare(i2c_dev->clk); return 0; @@ -2073,8 +2175,8 @@ static int __maybe_unused stm32f7_i2c_runtime_resume(struct device *dev) return 0; } -static int __maybe_unused -stm32f7_i2c_regs_backup(struct stm32f7_i2c_dev *i2c_dev) +#ifdef CONFIG_PM_SLEEP +static int stm32f7_i2c_regs_backup(struct stm32f7_i2c_dev *i2c_dev) { int ret; struct stm32f7_i2c_regs *backup_regs = &i2c_dev->backup_regs; @@ -2087,16 +2189,15 @@ stm32f7_i2c_regs_backup(struct stm32f7_i2c_dev *i2c_dev) backup_regs->cr2 = readl_relaxed(i2c_dev->base + STM32F7_I2C_CR2); backup_regs->oar1 = readl_relaxed(i2c_dev->base + STM32F7_I2C_OAR1); backup_regs->oar2 = readl_relaxed(i2c_dev->base + STM32F7_I2C_OAR2); - backup_regs->pecr = readl_relaxed(i2c_dev->base + STM32F7_I2C_PECR); backup_regs->tmgr = readl_relaxed(i2c_dev->base + STM32F7_I2C_TIMINGR); + stm32f7_i2c_write_fm_plus_bits(i2c_dev, false); pm_runtime_put_sync(i2c_dev->dev); return ret; } -static int __maybe_unused -stm32f7_i2c_regs_restore(struct stm32f7_i2c_dev *i2c_dev) +static int stm32f7_i2c_regs_restore(struct stm32f7_i2c_dev *i2c_dev) { u32 cr1; int ret; @@ -2120,48 +2221,55 @@ stm32f7_i2c_regs_restore(struct stm32f7_i2c_dev *i2c_dev) writel_relaxed(backup_regs->cr2, i2c_dev->base + STM32F7_I2C_CR2); writel_relaxed(backup_regs->oar1, i2c_dev->base + STM32F7_I2C_OAR1); writel_relaxed(backup_regs->oar2, i2c_dev->base + STM32F7_I2C_OAR2); - writel_relaxed(backup_regs->pecr, i2c_dev->base + STM32F7_I2C_PECR); + stm32f7_i2c_write_fm_plus_bits(i2c_dev, true); pm_runtime_put_sync(i2c_dev->dev); return ret; } -static int __maybe_unused stm32f7_i2c_suspend(struct device *dev) +static int stm32f7_i2c_suspend(struct device *dev) { struct stm32f7_i2c_dev *i2c_dev = dev_get_drvdata(dev); int ret; i2c_mark_adapter_suspended(&i2c_dev->adap); - ret = stm32f7_i2c_regs_backup(i2c_dev); - if (ret < 0) { - i2c_mark_adapter_resumed(&i2c_dev->adap); - return ret; - } - pinctrl_pm_select_sleep_state(dev); - pm_runtime_force_suspend(dev); + if (!device_may_wakeup(dev) && !dev->power.wakeup_path) { + ret = stm32f7_i2c_regs_backup(i2c_dev); + if (ret < 0) { + i2c_mark_adapter_resumed(&i2c_dev->adap); + return ret; + } + + pinctrl_pm_select_sleep_state(dev); + pm_runtime_force_suspend(dev); + } return 0; } -static int __maybe_unused stm32f7_i2c_resume(struct device *dev) +static int stm32f7_i2c_resume(struct device *dev) { struct stm32f7_i2c_dev *i2c_dev = dev_get_drvdata(dev); int ret; - ret = pm_runtime_force_resume(dev); - if (ret < 0) - return ret; - pinctrl_pm_select_default_state(dev); + if (!device_may_wakeup(dev) && !dev->power.wakeup_path) { + ret = pm_runtime_force_resume(dev); + if (ret < 0) + return ret; + pinctrl_pm_select_default_state(dev); + + ret = stm32f7_i2c_regs_restore(i2c_dev); + if (ret < 0) + return ret; + } - ret = stm32f7_i2c_regs_restore(i2c_dev); - if (ret < 0) - return ret; i2c_mark_adapter_resumed(&i2c_dev->adap); return 0; } +#endif static const struct dev_pm_ops stm32f7_i2c_pm_ops = { SET_RUNTIME_PM_OPS(stm32f7_i2c_runtime_suspend, @@ -2171,6 +2279,7 @@ static const struct dev_pm_ops stm32f7_i2c_pm_ops = { static const struct of_device_id stm32f7_i2c_match[] = { { .compatible = "st,stm32f7-i2c", .data = &stm32f7_setup}, + { .compatible = "st,stm32mp15-i2c", .data = &stm32mp15_setup}, {}, }; MODULE_DEVICE_TABLE(of, stm32f7_i2c_match); diff --git a/drivers/i2c/busses/i2c-stu300.c b/drivers/i2c/busses/i2c-stu300.c index 42e0a53e7fa4..ba6b60caa45e 100644 --- a/drivers/i2c/busses/i2c-stu300.c +++ b/drivers/i2c/busses/i2c-stu300.c @@ -132,7 +132,7 @@ enum stu300_error { #define NUM_ADDR_RESEND_ATTEMPTS 12 /* I2C clock speed, in Hz 0-400kHz*/ -static unsigned int scl_frequency = 100000; +static unsigned int scl_frequency = I2C_MAX_STANDARD_MODE_FREQ; module_param(scl_frequency, uint, 0644); /** @@ -497,7 +497,7 @@ static int stu300_set_clk(struct stu300_dev *dev, unsigned long clkrate) dev_dbg(&dev->pdev->dev, "Clock rate %lu Hz, I2C bus speed %d Hz " "virtbase %p\n", clkrate, dev->speed, dev->virtbase); - if (dev->speed > 100000) + if (dev->speed > I2C_MAX_STANDARD_MODE_FREQ) /* Fast Mode I2C */ val = ((clkrate/dev->speed) - 9)/3 + 1; else @@ -518,7 +518,7 @@ static int stu300_set_clk(struct stu300_dev *dev, unsigned long clkrate) return -EINVAL; } - if (dev->speed > 100000) { + if (dev->speed > I2C_MAX_STANDARD_MODE_FREQ) { /* CC6..CC0 */ stu300_wr8((val & I2C_CCR_CC_MASK) | I2C_CCR_FMSM, dev->virtbase + I2C_CCR); diff --git a/drivers/i2c/busses/i2c-sun6i-p2wi.c b/drivers/i2c/busses/i2c-sun6i-p2wi.c index 7c07ce116e38..e5293f0b3318 100644 --- a/drivers/i2c/busses/i2c-sun6i-p2wi.c +++ b/drivers/i2c/busses/i2c-sun6i-p2wi.c @@ -186,7 +186,7 @@ static int p2wi_probe(struct platform_device *pdev) struct device_node *np = dev->of_node; struct device_node *childnp; unsigned long parent_clk_freq; - u32 clk_freq = 100000; + u32 clk_freq = I2C_MAX_STANDARD_MODE_FREQ; struct resource *r; struct p2wi *p2wi; u32 slave_addr; diff --git a/drivers/i2c/busses/i2c-synquacer.c b/drivers/i2c/busses/i2c-synquacer.c index 86026798b4f7..9099d0a67ace 100644 --- a/drivers/i2c/busses/i2c-synquacer.c +++ b/drivers/i2c/busses/i2c-synquacer.c @@ -67,10 +67,10 @@ /* STANDARD MODE frequency */ #define SYNQUACER_I2C_CLK_MASTER_STD(rate) \ - DIV_ROUND_UP(DIV_ROUND_UP((rate), 100000) - 2, 2) + DIV_ROUND_UP(DIV_ROUND_UP((rate), I2C_MAX_STANDARD_MODE_FREQ) - 2, 2) /* FAST MODE frequency */ #define SYNQUACER_I2C_CLK_MASTER_FAST(rate) \ - DIV_ROUND_UP((DIV_ROUND_UP((rate), 400000) - 2) * 2, 3) + DIV_ROUND_UP((DIV_ROUND_UP((rate), I2C_MAX_FAST_MODE_FREQ) - 2) * 2, 3) /* (clkrate <= 18000000) */ /* calculate the value of CS bits in CCR register on standard mode */ @@ -602,7 +602,7 @@ static int synquacer_i2c_probe(struct platform_device *pdev) i2c->adapter.nr = pdev->id; init_completion(&i2c->completion); - if (bus_speed < 400000) + if (bus_speed < I2C_MAX_FAST_MODE_FREQ) i2c->speed_khz = SYNQUACER_I2C_SPEED_SM; else i2c->speed_khz = SYNQUACER_I2C_SPEED_FM; diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index cbc2ad49043e..4c4d17ddc96b 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -123,10 +123,6 @@ #define I2C_THIGH_SHIFT 8 #define I2C_INTERFACE_TIMING_1 0x98 -#define I2C_STANDARD_MODE 100000 -#define I2C_FAST_MODE 400000 -#define I2C_FAST_PLUS_MODE 1000000 - /* Packet header size in bytes */ #define I2C_PACKET_HEADER_SIZE 12 @@ -737,8 +733,8 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) I2C_CLK_DIVISOR_STD_FAST_MODE_SHIFT; i2c_writel(i2c_dev, clk_divisor, I2C_CLK_DIVISOR); - if (i2c_dev->bus_clk_rate > I2C_STANDARD_MODE && - i2c_dev->bus_clk_rate <= I2C_FAST_PLUS_MODE) { + if (i2c_dev->bus_clk_rate > I2C_MAX_STANDARD_MODE_FREQ && + i2c_dev->bus_clk_rate <= I2C_MAX_FAST_MODE_PLUS_FREQ) { tlow = i2c_dev->hw->tlow_fast_fastplus_mode; thigh = i2c_dev->hw->thigh_fast_fastplus_mode; tsu_thd = i2c_dev->hw->setup_hold_time_fast_fast_plus_mode; @@ -1341,7 +1337,7 @@ static void tegra_i2c_parse_dt(struct tegra_i2c_dev *i2c_dev) ret = of_property_read_u32(np, "clock-frequency", &i2c_dev->bus_clk_rate); if (ret) - i2c_dev->bus_clk_rate = 100000; /* default clock rate */ + i2c_dev->bus_clk_rate = I2C_MAX_STANDARD_MODE_FREQ; /* default clock rate */ multi_mode = of_property_read_bool(np, "multi-master"); i2c_dev->is_multimaster_mode = multi_mode; @@ -1640,12 +1636,12 @@ static int tegra_i2c_probe(struct platform_device *pdev) } } - if (i2c_dev->bus_clk_rate > I2C_FAST_MODE && - i2c_dev->bus_clk_rate <= I2C_FAST_PLUS_MODE) + if (i2c_dev->bus_clk_rate > I2C_MAX_FAST_MODE_FREQ && + i2c_dev->bus_clk_rate <= I2C_MAX_FAST_MODE_PLUS_FREQ) i2c_dev->clk_divisor_non_hs_mode = i2c_dev->hw->clk_divisor_fast_plus_mode; - else if (i2c_dev->bus_clk_rate > I2C_STANDARD_MODE && - i2c_dev->bus_clk_rate <= I2C_FAST_MODE) + else if (i2c_dev->bus_clk_rate > I2C_MAX_STANDARD_MODE_FREQ && + i2c_dev->bus_clk_rate <= I2C_MAX_FAST_MODE_FREQ) i2c_dev->clk_divisor_non_hs_mode = i2c_dev->hw->clk_divisor_fast_mode; else diff --git a/drivers/i2c/busses/i2c-thunderx-pcidrv.c b/drivers/i2c/busses/i2c-thunderx-pcidrv.c index 19f8eec38717..12c90aa0900e 100644 --- a/drivers/i2c/busses/i2c-thunderx-pcidrv.c +++ b/drivers/i2c/busses/i2c-thunderx-pcidrv.c @@ -118,6 +118,8 @@ static void thunder_i2c_clock_disable(struct device *dev, struct clk *clk) static int thunder_i2c_smbus_setup_of(struct octeon_i2c *i2c, struct device_node *node) { + struct i2c_client *ara; + if (!node) return -EINVAL; @@ -125,9 +127,12 @@ static int thunder_i2c_smbus_setup_of(struct octeon_i2c *i2c, if (!i2c->alert_data.irq) return -EINVAL; - i2c->ara = i2c_setup_smbus_alert(&i2c->adap, &i2c->alert_data); - if (!i2c->ara) - return -ENODEV; + ara = i2c_new_smbus_alert_device(&i2c->adap, &i2c->alert_data); + if (IS_ERR(ara)) + return PTR_ERR(ara); + + i2c->ara = ara; + return 0; } @@ -178,7 +183,7 @@ static int thunder_i2c_probe_pci(struct pci_dev *pdev, thunder_i2c_clock_enable(dev, i2c); ret = device_property_read_u32(dev, "clock-frequency", &i2c->twsi_freq); if (ret) - i2c->twsi_freq = 100000; + i2c->twsi_freq = I2C_MAX_STANDARD_MODE_FREQ; init_waitqueue_head(&i2c->queue); diff --git a/drivers/i2c/busses/i2c-uniphier-f.c b/drivers/i2c/busses/i2c-uniphier-f.c index 4241aac79e7e..2b258d54d68c 100644 --- a/drivers/i2c/busses/i2c-uniphier-f.c +++ b/drivers/i2c/busses/i2c-uniphier-f.c @@ -73,8 +73,6 @@ #define UNIPHIER_FI2C_BYTE_WISE BIT(3) #define UNIPHIER_FI2C_DEFER_STOP_COMP BIT(4) -#define UNIPHIER_FI2C_DEFAULT_SPEED 100000 -#define UNIPHIER_FI2C_MAX_SPEED 400000 #define UNIPHIER_FI2C_FIFO_SIZE 8 struct uniphier_fi2c_priv { @@ -537,9 +535,9 @@ static int uniphier_fi2c_probe(struct platform_device *pdev) } if (of_property_read_u32(dev->of_node, "clock-frequency", &bus_speed)) - bus_speed = UNIPHIER_FI2C_DEFAULT_SPEED; + bus_speed = I2C_MAX_STANDARD_MODE_FREQ; - if (!bus_speed || bus_speed > UNIPHIER_FI2C_MAX_SPEED) { + if (!bus_speed || bus_speed > I2C_MAX_FAST_MODE_FREQ) { dev_err(dev, "invalid clock-frequency %d\n", bus_speed); return -EINVAL; } diff --git a/drivers/i2c/busses/i2c-uniphier.c b/drivers/i2c/busses/i2c-uniphier.c index 0270090c0360..668b1fa2b0ef 100644 --- a/drivers/i2c/busses/i2c-uniphier.c +++ b/drivers/i2c/busses/i2c-uniphier.c @@ -35,9 +35,6 @@ #define UNIPHIER_I2C_NOISE 0x1c /* noise filter control */ #define UNIPHIER_I2C_SETUP 0x20 /* setup time control */ -#define UNIPHIER_I2C_DEFAULT_SPEED 100000 -#define UNIPHIER_I2C_MAX_SPEED 400000 - struct uniphier_i2c_priv { struct completion comp; struct i2c_adapter adap; @@ -333,9 +330,9 @@ static int uniphier_i2c_probe(struct platform_device *pdev) } if (of_property_read_u32(dev->of_node, "clock-frequency", &bus_speed)) - bus_speed = UNIPHIER_I2C_DEFAULT_SPEED; + bus_speed = I2C_MAX_STANDARD_MODE_FREQ; - if (!bus_speed || bus_speed > UNIPHIER_I2C_MAX_SPEED) { + if (!bus_speed || bus_speed > I2C_MAX_FAST_MODE_FREQ) { dev_err(dev, "invalid clock-frequency %d\n", bus_speed); return -EINVAL; } diff --git a/drivers/i2c/busses/i2c-wmt.c b/drivers/i2c/busses/i2c-wmt.c index 524017f7034e..88f5aafdce5b 100644 --- a/drivers/i2c/busses/i2c-wmt.c +++ b/drivers/i2c/busses/i2c-wmt.c @@ -399,7 +399,7 @@ static int wmt_i2c_probe(struct platform_device *pdev) i2c_dev->mode = I2C_MODE_STANDARD; err = of_property_read_u32(np, "clock-frequency", &clk_rate); - if ((!err) && (clk_rate == 400000)) + if (!err && (clk_rate == I2C_MAX_FAST_MODE_FREQ)) i2c_dev->mode = I2C_MODE_FAST; i2c_dev->dev = &pdev->dev; diff --git a/drivers/i2c/busses/i2c-xlp9xx.c b/drivers/i2c/busses/i2c-xlp9xx.c index 8a873975cf12..391c878a7cdc 100644 --- a/drivers/i2c/busses/i2c-xlp9xx.c +++ b/drivers/i2c/busses/i2c-xlp9xx.c @@ -71,8 +71,6 @@ #define XLP9XX_I2C_SLAVEADDR_ADDR_SHIFT 1 #define XLP9XX_I2C_IP_CLK_FREQ 133000000UL -#define XLP9XX_I2C_DEFAULT_FREQ 100000 -#define XLP9XX_I2C_HIGH_FREQ 400000 #define XLP9XX_I2C_FIFO_SIZE 0x80U #define XLP9XX_I2C_TIMEOUT_MS 1000 #define XLP9XX_I2C_BUSY_TIMEOUT 50 @@ -476,12 +474,12 @@ static int xlp9xx_i2c_get_frequency(struct platform_device *pdev, err = device_property_read_u32(&pdev->dev, "clock-frequency", &freq); if (err) { - freq = XLP9XX_I2C_DEFAULT_FREQ; + freq = I2C_MAX_STANDARD_MODE_FREQ; dev_dbg(&pdev->dev, "using default frequency %u\n", freq); - } else if (freq == 0 || freq > XLP9XX_I2C_HIGH_FREQ) { + } else if (freq == 0 || freq > I2C_MAX_FAST_MODE_FREQ) { dev_warn(&pdev->dev, "invalid frequency %u, using default\n", freq); - freq = XLP9XX_I2C_DEFAULT_FREQ; + freq = I2C_MAX_STANDARD_MODE_FREQ; } priv->clk_hz = freq; @@ -491,12 +489,16 @@ static int xlp9xx_i2c_get_frequency(struct platform_device *pdev, static int xlp9xx_i2c_smbus_setup(struct xlp9xx_i2c_dev *priv, struct platform_device *pdev) { + struct i2c_client *ara; + if (!priv->alert_data.irq) return -EINVAL; - priv->ara = i2c_setup_smbus_alert(&priv->adapter, &priv->alert_data); - if (!priv->ara) - return -ENODEV; + ara = i2c_new_smbus_alert_device(&priv->adapter, &priv->alert_data); + if (IS_ERR(ara)) + return PTR_ERR(ara); + + priv->ara = ara; return 0; } diff --git a/drivers/i2c/busses/i2c-xlr.c b/drivers/i2c/busses/i2c-xlr.c index 34cd4b308540..282f161a8b08 100644 --- a/drivers/i2c/busses/i2c-xlr.c +++ b/drivers/i2c/busses/i2c-xlr.c @@ -404,7 +404,7 @@ static int xlr_i2c_probe(struct platform_device *pdev) if (of_property_read_u32(pdev->dev.of_node, "clock-frequency", &busfreq)) - busfreq = 100000; + busfreq = I2C_MAX_STANDARD_MODE_FREQ; clk = devm_clk_get(&pdev->dev, NULL); if (!IS_ERR(clk)) { diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index 8b0ff780919b..c8f42f2037cb 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -318,7 +318,7 @@ static acpi_status i2c_acpi_lookup_speed(acpi_handle handle, u32 level, lookup->min_speed = lookup->speed; if (acpi_match_device_ids(adev, i2c_acpi_force_400khz_device_ids) == 0) - lookup->force_speed = 400000; + lookup->force_speed = I2C_MAX_FAST_MODE_FREQ; return AE_OK; } diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index cefad0881942..5cc0b0ec5570 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -1593,32 +1593,30 @@ EXPORT_SYMBOL(i2c_del_adapter); * @dev: The device to scan for I2C timing properties * @t: the i2c_timings struct to be filled with values * @use_defaults: bool to use sane defaults derived from the I2C specification - * when properties are not found, otherwise use 0 + * when properties are not found, otherwise don't update * * Scan the device for the generic I2C properties describing timing parameters * for the signal and fill the given struct with the results. If a property was * not found and use_defaults was true, then maximum timings are assumed which * are derived from the I2C specification. If use_defaults is not used, the - * results will be 0, so drivers can apply their own defaults later. The latter - * is mainly intended for avoiding regressions of existing drivers which want - * to switch to this function. New drivers almost always should use the defaults. + * results will be as before, so drivers can apply their own defaults before + * calling this helper. The latter is mainly intended for avoiding regressions + * of existing drivers which want to switch to this function. New drivers + * almost always should use the defaults. */ - void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_defaults) { int ret; - memset(t, 0, sizeof(*t)); - ret = device_property_read_u32(dev, "clock-frequency", &t->bus_freq_hz); if (ret && use_defaults) - t->bus_freq_hz = 100000; + t->bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; ret = device_property_read_u32(dev, "i2c-scl-rising-time-ns", &t->scl_rise_ns); if (ret && use_defaults) { - if (t->bus_freq_hz <= 100000) + if (t->bus_freq_hz <= I2C_MAX_STANDARD_MODE_FREQ) t->scl_rise_ns = 1000; - else if (t->bus_freq_hz <= 400000) + else if (t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ) t->scl_rise_ns = 300; else t->scl_rise_ns = 120; @@ -1626,25 +1624,31 @@ void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_de ret = device_property_read_u32(dev, "i2c-scl-falling-time-ns", &t->scl_fall_ns); if (ret && use_defaults) { - if (t->bus_freq_hz <= 400000) + if (t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ) t->scl_fall_ns = 300; else t->scl_fall_ns = 120; } - device_property_read_u32(dev, "i2c-scl-internal-delay-ns", &t->scl_int_delay_ns); + ret = device_property_read_u32(dev, "i2c-scl-internal-delay-ns", &t->scl_int_delay_ns); + if (ret && use_defaults) + t->scl_int_delay_ns = 0; ret = device_property_read_u32(dev, "i2c-sda-falling-time-ns", &t->sda_fall_ns); if (ret && use_defaults) t->sda_fall_ns = t->scl_fall_ns; - device_property_read_u32(dev, "i2c-sda-hold-time-ns", &t->sda_hold_ns); + ret = device_property_read_u32(dev, "i2c-sda-hold-time-ns", &t->sda_hold_ns); + if (ret && use_defaults) + t->sda_hold_ns = 0; - device_property_read_u32(dev, "i2c-digital-filter-width-ns", - &t->digital_filter_width_ns); + ret = device_property_read_u32(dev, "i2c-digital-filter-width-ns", &t->digital_filter_width_ns); + if (ret && use_defaults) + t->digital_filter_width_ns = 0; - device_property_read_u32(dev, "i2c-analog-filter-cutoff-frequency", - &t->analog_filter_cutoff_freq_hz); + ret = device_property_read_u32(dev, "i2c-analog-filter-cutoff-frequency", &t->analog_filter_cutoff_freq_hz); + if (ret && use_defaults) + t->analog_filter_cutoff_freq_hz = 0; } EXPORT_SYMBOL_GPL(i2c_parse_fw_timings); diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c index 3ac426a8ab5a..b34d2ff06931 100644 --- a/drivers/i2c/i2c-core-smbus.c +++ b/drivers/i2c/i2c-core-smbus.c @@ -666,7 +666,7 @@ s32 i2c_smbus_read_i2c_block_data_or_emulated(const struct i2c_client *client, EXPORT_SYMBOL(i2c_smbus_read_i2c_block_data_or_emulated); /** - * i2c_setup_smbus_alert - Setup SMBus alert support + * i2c_new_smbus_alert_device - get ara client for SMBus alert support * @adapter: the target adapter * @setup: setup data for the SMBus alert handler * Context: can sleep @@ -676,31 +676,25 @@ EXPORT_SYMBOL(i2c_smbus_read_i2c_block_data_or_emulated); * Handling can be done either through our IRQ handler, or by the * adapter (from its handler, periodic polling, or whatever). * - * NOTE that if we manage the IRQ, we *MUST* know if it's level or - * edge triggered in order to hand it to the workqueue correctly. - * If triggering the alert seems to wedge the system, you probably - * should have said it's level triggered. - * * This returns the ara client, which should be saved for later use with - * i2c_handle_smbus_alert() and ultimately i2c_unregister_device(); or NULL - * to indicate an error. + * i2c_handle_smbus_alert() and ultimately i2c_unregister_device(); or an + * ERRPTR to indicate an error. */ -struct i2c_client *i2c_setup_smbus_alert(struct i2c_adapter *adapter, - struct i2c_smbus_alert_setup *setup) +struct i2c_client *i2c_new_smbus_alert_device(struct i2c_adapter *adapter, + struct i2c_smbus_alert_setup *setup) { struct i2c_board_info ara_board_info = { I2C_BOARD_INFO("smbus_alert", 0x0c), .platform_data = setup, }; - return i2c_new_device(adapter, &ara_board_info); + return i2c_new_client_device(adapter, &ara_board_info); } -EXPORT_SYMBOL_GPL(i2c_setup_smbus_alert); +EXPORT_SYMBOL_GPL(i2c_new_smbus_alert_device); #if IS_ENABLED(CONFIG_I2C_SMBUS) && IS_ENABLED(CONFIG_OF) int of_i2c_setup_smbus_alert(struct i2c_adapter *adapter) { - struct i2c_client *client; int irq; irq = of_property_match_string(adapter->dev.of_node, "interrupt-names", @@ -710,11 +704,7 @@ int of_i2c_setup_smbus_alert(struct i2c_adapter *adapter) else if (irq < 0) return irq; - client = i2c_setup_smbus_alert(adapter, NULL); - if (!client) - return -ENODEV; - - return 0; + return PTR_ERR_OR_ZERO(i2c_new_smbus_alert_device(adapter, NULL)); } EXPORT_SYMBOL_GPL(of_i2c_setup_smbus_alert); #endif diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c index 2ea4585d18c5..da020acc9bbd 100644 --- a/drivers/i2c/i2c-dev.c +++ b/drivers/i2c/i2c-dev.c @@ -15,6 +15,7 @@ /* The I2C_RDWR ioctl code is written by Kolja Waschk */ #include +#include #include #include #include @@ -27,7 +28,6 @@ #include #include #include -#include /* * An i2c_dev represents an i2c_adapter ... an I2C or SMBus master, not a @@ -40,7 +40,7 @@ struct i2c_dev { struct list_head list; struct i2c_adapter *adap; - struct device *dev; + struct device dev; struct cdev cdev; }; @@ -84,12 +84,14 @@ static struct i2c_dev *get_free_i2c_dev(struct i2c_adapter *adap) return i2c_dev; } -static void put_i2c_dev(struct i2c_dev *i2c_dev) +static void put_i2c_dev(struct i2c_dev *i2c_dev, bool del_cdev) { spin_lock(&i2c_dev_list_lock); list_del(&i2c_dev->list); spin_unlock(&i2c_dev_list_lock); - kfree(i2c_dev); + if (del_cdev) + cdev_device_del(&i2c_dev->cdev, &i2c_dev->dev); + put_device(&i2c_dev->dev); } static ssize_t name_show(struct device *dev, @@ -628,6 +630,14 @@ static const struct file_operations i2cdev_fops = { static struct class *i2c_dev_class; +static void i2cdev_dev_release(struct device *dev) +{ + struct i2c_dev *i2c_dev; + + i2c_dev = container_of(dev, struct i2c_dev, dev); + kfree(i2c_dev); +} + static int i2cdev_attach_adapter(struct device *dev, void *dummy) { struct i2c_adapter *adap; @@ -644,27 +654,23 @@ static int i2cdev_attach_adapter(struct device *dev, void *dummy) cdev_init(&i2c_dev->cdev, &i2cdev_fops); i2c_dev->cdev.owner = THIS_MODULE; - res = cdev_add(&i2c_dev->cdev, MKDEV(I2C_MAJOR, adap->nr), 1); - if (res) - goto error_cdev; - /* register this i2c device with the driver core */ - i2c_dev->dev = device_create(i2c_dev_class, &adap->dev, - MKDEV(I2C_MAJOR, adap->nr), NULL, - "i2c-%d", adap->nr); - if (IS_ERR(i2c_dev->dev)) { - res = PTR_ERR(i2c_dev->dev); - goto error; + device_initialize(&i2c_dev->dev); + i2c_dev->dev.devt = MKDEV(I2C_MAJOR, adap->nr); + i2c_dev->dev.class = i2c_dev_class; + i2c_dev->dev.parent = &adap->dev; + i2c_dev->dev.release = i2cdev_dev_release; + dev_set_name(&i2c_dev->dev, "i2c-%d", adap->nr); + + res = cdev_device_add(&i2c_dev->cdev, &i2c_dev->dev); + if (res) { + put_i2c_dev(i2c_dev, false); + return res; } pr_debug("i2c-dev: adapter [%s] registered as minor %d\n", adap->name, adap->nr); return 0; -error: - cdev_del(&i2c_dev->cdev); -error_cdev: - put_i2c_dev(i2c_dev); - return res; } static int i2cdev_detach_adapter(struct device *dev, void *dummy) @@ -680,9 +686,7 @@ static int i2cdev_detach_adapter(struct device *dev, void *dummy) if (!i2c_dev) /* attach_adapter must have failed */ return 0; - cdev_del(&i2c_dev->cdev); - put_i2c_dev(i2c_dev); - device_destroy(i2c_dev_class, MKDEV(I2C_MAJOR, adap->nr)); + put_i2c_dev(i2c_dev, true); pr_debug("i2c-dev: adapter [%s] unregistered\n", adap->name); return 0; diff --git a/drivers/i2c/i2c-slave-eeprom.c b/drivers/i2c/i2c-slave-eeprom.c index db9763cb4dae..cb415b10642f 100644 --- a/drivers/i2c/i2c-slave-eeprom.c +++ b/drivers/i2c/i2c-slave-eeprom.c @@ -96,7 +96,7 @@ static ssize_t i2c_slave_eeprom_bin_read(struct file *filp, struct kobject *kobj struct eeprom_data *eeprom; unsigned long flags; - eeprom = dev_get_drvdata(container_of(kobj, struct device, kobj)); + eeprom = dev_get_drvdata(kobj_to_dev(kobj)); spin_lock_irqsave(&eeprom->buffer_lock, flags); memcpy(buf, &eeprom->buffer[off], count); @@ -111,7 +111,7 @@ static ssize_t i2c_slave_eeprom_bin_write(struct file *filp, struct kobject *kob struct eeprom_data *eeprom; unsigned long flags; - eeprom = dev_get_drvdata(container_of(kobj, struct device, kobj)); + eeprom = dev_get_drvdata(kobj_to_dev(kobj)); spin_lock_irqsave(&eeprom->buffer_lock, flags); memcpy(&eeprom->buffer[off], buf, count); diff --git a/drivers/i2c/i2c-smbus.c b/drivers/i2c/i2c-smbus.c index 7e2f5d0eacdb..809bcf8387d0 100644 --- a/drivers/i2c/i2c-smbus.c +++ b/drivers/i2c/i2c-smbus.c @@ -184,7 +184,7 @@ static struct i2c_driver smbalert_driver = { * corresponding I2C device driver's alert function. * * It is assumed that ara is a valid i2c client previously returned by - * i2c_setup_smbus_alert(). + * i2c_new_smbus_alert_device(). */ int i2c_handle_smbus_alert(struct i2c_client *ara) { diff --git a/drivers/macintosh/therm_windtunnel.c b/drivers/macintosh/therm_windtunnel.c index a0d87ed9da69..f55f6adf5e5f 100644 --- a/drivers/macintosh/therm_windtunnel.c +++ b/drivers/macintosh/therm_windtunnel.c @@ -323,7 +323,7 @@ static void do_attach(struct i2c_adapter *adapter) of_node_put(np); } else { strlcpy(info.type, "MAC,ds1775", I2C_NAME_SIZE); - i2c_new_probed_device(adapter, &info, scan_ds1775, NULL); + i2c_new_scanned_device(adapter, &info, scan_ds1775, NULL); } np = of_find_compatible_node(adapter->dev.of_node, NULL, "MAC,adm1030"); @@ -331,7 +331,7 @@ static void do_attach(struct i2c_adapter *adapter) of_node_put(np); } else { strlcpy(info.type, "MAC,adm1030", I2C_NAME_SIZE); - i2c_new_probed_device(adapter, &info, scan_adm1030, NULL); + i2c_new_scanned_device(adapter, &info, scan_adm1030, NULL); } } diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c index 282c9ef68ed2..9ff18d4961ce 100644 --- a/drivers/misc/eeprom/at24.c +++ b/drivers/misc/eeprom/at24.c @@ -227,6 +227,7 @@ MODULE_DEVICE_TABLE(of, at24_of_match); static const struct acpi_device_id at24_acpi_ids[] = { { "INT3499", (kernel_ulong_t)&at24_data_INT3499 }, + { "TPF0001", (kernel_ulong_t)&at24_data_24c1024 }, { /* END OF LIST */ } }; MODULE_DEVICE_TABLE(acpi, at24_acpi_ids); diff --git a/drivers/mtd/chips/cfi_cmdset_0001.c b/drivers/mtd/chips/cfi_cmdset_0001.c index 00a79489067c..142c0f9485fe 100644 --- a/drivers/mtd/chips/cfi_cmdset_0001.c +++ b/drivers/mtd/chips/cfi_cmdset_0001.c @@ -834,7 +834,7 @@ static int chip_ready (struct map_info *map, struct flchip *chip, unsigned long /* Someone else might have been playing with it. */ return -EAGAIN; } - /* Fall through */ + fallthrough; case FL_READY: case FL_CFI_QUERY: case FL_JEDEC_QUERY: @@ -907,7 +907,7 @@ static int chip_ready (struct map_info *map, struct flchip *chip, unsigned long /* Only if there's no operation suspended... */ if (mode == FL_READY && chip->oldstate == FL_READY) return 0; - /* Fall through */ + fallthrough; default: sleep: set_current_state(TASK_UNINTERRUPTIBLE); diff --git a/drivers/mtd/chips/cfi_cmdset_0002.c b/drivers/mtd/chips/cfi_cmdset_0002.c index 04b383bc3947..a1f3e1031c3d 100644 --- a/drivers/mtd/chips/cfi_cmdset_0002.c +++ b/drivers/mtd/chips/cfi_cmdset_0002.c @@ -966,8 +966,7 @@ static int get_chip(struct map_info *map, struct flchip *chip, unsigned long adr /* Only if there's no operation suspended... */ if (mode == FL_READY && chip->oldstate == FL_READY) return 0; - /* fall through */ - + fallthrough; default: sleep: set_current_state(TASK_UNINTERRUPTIBLE); @@ -2935,7 +2934,7 @@ static void cfi_amdstd_sync (struct mtd_info *mtd) * as the whole point is that nobody can do anything * with the chip now anyway. */ - /* fall through */ + fallthrough; case FL_SYNCING: mutex_unlock(&chip->mutex); break; diff --git a/drivers/mtd/chips/cfi_cmdset_0020.c b/drivers/mtd/chips/cfi_cmdset_0020.c index 54edae63b92d..270322bca221 100644 --- a/drivers/mtd/chips/cfi_cmdset_0020.c +++ b/drivers/mtd/chips/cfi_cmdset_0020.c @@ -324,8 +324,7 @@ static inline int do_read_onechip(struct map_info *map, struct flchip *chip, lof case FL_JEDEC_QUERY: map_write(map, CMD(0x70), cmd_addr); chip->state = FL_STATUS; - /* Fall through */ - + fallthrough; case FL_STATUS: status = map_read(map, cmd_addr); if (map_word_andequal(map, status, status_OK, status_OK)) { @@ -462,8 +461,7 @@ static int do_write_buffer(struct map_info *map, struct flchip *chip, #ifdef DEBUG_CFI_FEATURES printk("%s: 1 status[%x]\n", __func__, map_read(map, cmd_adr)); #endif - /* Fall through */ - + fallthrough; case FL_STATUS: status = map_read(map, cmd_adr); if (map_word_andequal(map, status, status_OK, status_OK)) @@ -756,8 +754,7 @@ static inline int do_erase_oneblock(struct map_info *map, struct flchip *chip, u case FL_READY: map_write(map, CMD(0x70), adr); chip->state = FL_STATUS; - /* Fall through */ - + fallthrough; case FL_STATUS: status = map_read(map, adr); if (map_word_andequal(map, status, status_OK, status_OK)) @@ -998,7 +995,7 @@ static void cfi_staa_sync (struct mtd_info *mtd) * as the whole point is that nobody can do anything * with the chip now anyway. */ - /* Fall through */ + fallthrough; case FL_SYNCING: mutex_unlock(&chip->mutex); break; @@ -1054,8 +1051,7 @@ static inline int do_lock_oneblock(struct map_info *map, struct flchip *chip, un case FL_READY: map_write(map, CMD(0x70), adr); chip->state = FL_STATUS; - /* Fall through */ - + fallthrough; case FL_STATUS: status = map_read(map, adr); if (map_word_andequal(map, status, status_OK, status_OK)) @@ -1201,8 +1197,7 @@ static inline int do_unlock_oneblock(struct map_info *map, struct flchip *chip, case FL_READY: map_write(map, CMD(0x70), adr); chip->state = FL_STATUS; - /* Fall through */ - + fallthrough; case FL_STATUS: status = map_read(map, adr); if (map_word_andequal(map, status, status_OK, status_OK)) diff --git a/drivers/mtd/chips/cfi_util.c b/drivers/mtd/chips/cfi_util.c index e2d4db05aeb3..99b7986002f0 100644 --- a/drivers/mtd/chips/cfi_util.c +++ b/drivers/mtd/chips/cfi_util.c @@ -109,13 +109,13 @@ map_word cfi_build_cmd(u_long cmd, struct map_info *map, struct cfi_private *cfi case 8: onecmd |= (onecmd << (chip_mode * 32)); #endif - /* fall through */ + fallthrough; case 4: onecmd |= (onecmd << (chip_mode * 16)); - /* fall through */ + fallthrough; case 2: onecmd |= (onecmd << (chip_mode * 8)); - /* fall through */ + fallthrough; case 1: ; } @@ -165,13 +165,13 @@ unsigned long cfi_merge_status(map_word val, struct map_info *map, case 8: res |= (onestat >> (chip_mode * 32)); #endif - /* fall through */ + fallthrough; case 4: res |= (onestat >> (chip_mode * 16)); - /* fall through */ + fallthrough; case 2: res |= (onestat >> (chip_mode * 8)); - /* fall through */ + fallthrough; case 1: ; } diff --git a/drivers/mtd/devices/block2mtd.c b/drivers/mtd/devices/block2mtd.c index 36aa082f6db0..c08721b11642 100644 --- a/drivers/mtd/devices/block2mtd.c +++ b/drivers/mtd/devices/block2mtd.c @@ -329,10 +329,10 @@ static int ustrtoul(const char *cp, char **endp, unsigned int base) switch (**endp) { case 'G' : result *= 1024; - /* fall through */ + fallthrough; case 'M': result *= 1024; - /* fall through */ + fallthrough; case 'K': case 'k': result *= 1024; diff --git a/drivers/mtd/devices/phram.c b/drivers/mtd/devices/phram.c index 931e5c2481b5..087b5e86d1bf 100644 --- a/drivers/mtd/devices/phram.c +++ b/drivers/mtd/devices/phram.c @@ -148,10 +148,10 @@ static int parse_num64(uint64_t *num64, char *token) switch (token[len - 2]) { case 'G': shift += 10; - /* fall through */ + fallthrough; case 'M': shift += 10; - /* fall through */ + fallthrough; case 'k': shift += 10; token[len - 2] = 0; @@ -243,22 +243,25 @@ static int phram_setup(const char *val) ret = parse_num64(&start, token[1]); if (ret) { - kfree(name); parse_err("illegal start address\n"); + goto error; } ret = parse_num64(&len, token[2]); if (ret) { - kfree(name); parse_err("illegal device length\n"); + goto error; } ret = register_device(name, start, len); - if (!ret) - pr_info("%s device: %#llx at %#llx\n", name, len, start); - else - kfree(name); + if (ret) + goto error; + pr_info("%s device: %#llx at %#llx\n", name, len, start); + return 0; + +error: + kfree(name); return ret; } diff --git a/drivers/mtd/hyperbus/hbmc-am654.c b/drivers/mtd/hyperbus/hbmc-am654.c index 08d543b124cd..f350a0809f88 100644 --- a/drivers/mtd/hyperbus/hbmc-am654.c +++ b/drivers/mtd/hyperbus/hbmc-am654.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -57,8 +58,10 @@ static const struct hyperbus_ops am654_hbmc_ops = { static int am654_hbmc_probe(struct platform_device *pdev) { + struct device_node *np = pdev->dev.of_node; struct device *dev = &pdev->dev; struct am654_hbmc_priv *priv; + struct resource res; int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); @@ -67,6 +70,10 @@ static int am654_hbmc_probe(struct platform_device *pdev) platform_set_drvdata(pdev, priv); + ret = of_address_to_resource(np, 0, &res); + if (ret) + return ret; + if (of_property_read_bool(dev->of_node, "mux-controls")) { struct mux_control *control = devm_mux_control_get(dev, NULL); @@ -88,6 +95,11 @@ static int am654_hbmc_probe(struct platform_device *pdev) goto disable_pm; } + priv->hbdev.map.size = resource_size(&res); + priv->hbdev.map.virt = devm_ioremap_resource(dev, &res); + if (IS_ERR(priv->hbdev.map.virt)) + return PTR_ERR(priv->hbdev.map.virt); + priv->ctlr.dev = dev; priv->ctlr.ops = &am654_hbmc_ops; priv->hbdev.ctlr = &priv->ctlr; diff --git a/drivers/mtd/hyperbus/hyperbus-core.c b/drivers/mtd/hyperbus/hyperbus-core.c index 6af9ea34117d..32685e8dd278 100644 --- a/drivers/mtd/hyperbus/hyperbus-core.c +++ b/drivers/mtd/hyperbus/hyperbus-core.c @@ -10,7 +10,6 @@ #include #include #include -#include #include static struct hyperbus_device *map_to_hbdev(struct map_info *map) @@ -62,7 +61,6 @@ int hyperbus_register_device(struct hyperbus_device *hbdev) struct hyperbus_ctlr *ctlr; struct device_node *np; struct map_info *map; - struct resource res; struct device *dev; int ret; @@ -73,22 +71,15 @@ int hyperbus_register_device(struct hyperbus_device *hbdev) np = hbdev->np; ctlr = hbdev->ctlr; - if (!of_device_is_compatible(np, "cypress,hyperflash")) + if (!of_device_is_compatible(np, "cypress,hyperflash")) { + dev_err(ctlr->dev, "\"cypress,hyperflash\" compatible missing\n"); return -ENODEV; + } hbdev->memtype = HYPERFLASH; - ret = of_address_to_resource(np, 0, &res); - if (ret) - return ret; - dev = ctlr->dev; map = &hbdev->map; - map->size = resource_size(&res); - map->virt = devm_ioremap_resource(dev, &res); - if (IS_ERR(map->virt)) - return PTR_ERR(map->virt); - map->name = dev_name(dev); map->bankwidth = 2; map->device_node = np; diff --git a/drivers/mtd/inftlmount.c b/drivers/mtd/inftlmount.c index 54b176d4319f..af16d3485de0 100644 --- a/drivers/mtd/inftlmount.c +++ b/drivers/mtd/inftlmount.c @@ -130,7 +130,7 @@ static int find_boot_record(struct INFTLrecord *inftl) " NoOfBootImageBlocks = %d\n" " NoOfBinaryPartitions = %d\n" " NoOfBDTLPartitions = %d\n" - " BlockMultiplerBits = %d\n" + " BlockMultiplierBits = %d\n" " FormatFlgs = %d\n" " OsakVersion = 0x%x\n" " PercentUsed = %d\n", diff --git a/drivers/mtd/lpddr/lpddr_cmds.c b/drivers/mtd/lpddr/lpddr_cmds.c index 1efc643c9871..fb1cbc9a2870 100644 --- a/drivers/mtd/lpddr/lpddr_cmds.c +++ b/drivers/mtd/lpddr/lpddr_cmds.c @@ -68,7 +68,6 @@ struct mtd_info *lpddr_cmdset(struct map_info *map) shared = kmalloc_array(lpddr->numchips, sizeof(struct flchip_shared), GFP_KERNEL); if (!shared) { - kfree(lpddr); kfree(mtd); return NULL; } @@ -305,8 +304,7 @@ static int chip_ready(struct map_info *map, struct flchip *chip, int mode) /* Only if there's no operation suspended... */ if (mode == FL_READY && chip->oldstate == FL_READY) return 0; - /* fall through */ - + fallthrough; default: sleep: set_current_state(TASK_UNINTERRUPTIBLE); diff --git a/drivers/mtd/maps/sa1100-flash.c b/drivers/mtd/maps/sa1100-flash.c index 47602af4ee34..d3d4e987c163 100644 --- a/drivers/mtd/maps/sa1100-flash.c +++ b/drivers/mtd/maps/sa1100-flash.c @@ -34,7 +34,7 @@ struct sa_subdev_info { struct sa_info { struct mtd_info *mtd; int num_subdev; - struct sa_subdev_info subdev[0]; + struct sa_subdev_info subdev[]; }; static DEFINE_SPINLOCK(sa1100_vpp_lock); @@ -81,8 +81,7 @@ static int sa1100_probe_subdev(struct sa_subdev_info *subdev, struct resource *r default: printk(KERN_WARNING "SA1100 flash: unknown base address " "0x%08lx, assuming CS0\n", phys); - /* Fall through */ - + fallthrough; case SA1100_CS0_PHYS: subdev->map.bankwidth = (MSC0 & MSC_RBW) ? 2 : 4; break; diff --git a/drivers/mtd/mtdblock.c b/drivers/mtd/mtdblock.c index c06b5322d470..078e0f67377d 100644 --- a/drivers/mtd/mtdblock.c +++ b/drivers/mtd/mtdblock.c @@ -294,12 +294,13 @@ static void mtdblock_release(struct mtd_blktrans_dev *mbd) static int mtdblock_flush(struct mtd_blktrans_dev *dev) { struct mtdblk_dev *mtdblk = container_of(dev, struct mtdblk_dev, mbd); + int ret; mutex_lock(&mtdblk->cache_mutex); - write_cached_data(mtdblk); + ret = write_cached_data(mtdblk); mutex_unlock(&mtdblk->cache_mutex); mtd_sync(dev->mtd); - return 0; + return ret; } static void mtdblock_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd) diff --git a/drivers/mtd/mtdchar.c b/drivers/mtd/mtdchar.c index b841008a9eb7..c5935b2f9cd1 100644 --- a/drivers/mtd/mtdchar.c +++ b/drivers/mtd/mtdchar.c @@ -349,6 +349,7 @@ static int mtdchar_writeoob(struct file *file, struct mtd_info *mtd, uint64_t start, uint32_t length, void __user *ptr, uint32_t __user *retp) { + struct mtd_info *master = mtd_get_master(mtd); struct mtd_file_info *mfi = file->private_data; struct mtd_oob_ops ops = {}; uint32_t retlen; @@ -360,7 +361,7 @@ static int mtdchar_writeoob(struct file *file, struct mtd_info *mtd, if (length > 4096) return -EINVAL; - if (!mtd->_write_oob) + if (!master->_write_oob) return -EOPNOTSUPP; ops.ooblen = length; @@ -586,6 +587,7 @@ static int mtdchar_blkpg_ioctl(struct mtd_info *mtd, static int mtdchar_write_ioctl(struct mtd_info *mtd, struct mtd_write_req __user *argp) { + struct mtd_info *master = mtd_get_master(mtd); struct mtd_write_req req; struct mtd_oob_ops ops = {}; const void __user *usr_data, *usr_oob; @@ -597,9 +599,8 @@ static int mtdchar_write_ioctl(struct mtd_info *mtd, usr_data = (const void __user *)(uintptr_t)req.usr_data; usr_oob = (const void __user *)(uintptr_t)req.usr_oob; - if (!mtd->_write_oob) + if (!master->_write_oob) return -EOPNOTSUPP; - ops.mode = req.mode; ops.len = (size_t)req.len; ops.ooblen = (size_t)req.ooblen; @@ -635,6 +636,7 @@ static int mtdchar_ioctl(struct file *file, u_int cmd, u_long arg) { struct mtd_file_info *mfi = file->private_data; struct mtd_info *mtd = mfi->mtd; + struct mtd_info *master = mtd_get_master(mtd); void __user *argp = (void __user *)arg; int ret = 0; struct mtd_info_user info; @@ -824,7 +826,7 @@ static int mtdchar_ioctl(struct file *file, u_int cmd, u_long arg) { struct nand_oobinfo oi; - if (!mtd->ooblayout) + if (!master->ooblayout) return -EOPNOTSUPP; ret = get_oobinfo(mtd, &oi); @@ -918,7 +920,7 @@ static int mtdchar_ioctl(struct file *file, u_int cmd, u_long arg) { struct nand_ecclayout_user *usrlay; - if (!mtd->ooblayout) + if (!master->ooblayout) return -EOPNOTSUPP; usrlay = kmalloc(sizeof(*usrlay), GFP_KERNEL); diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index 5fac4355b9c2..2916674208b3 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -456,13 +456,14 @@ static int mtd_reboot_notifier(struct notifier_block *n, unsigned long state, int mtd_wunit_to_pairing_info(struct mtd_info *mtd, int wunit, struct mtd_pairing_info *info) { - int npairs = mtd_wunit_per_eb(mtd) / mtd_pairing_groups(mtd); + struct mtd_info *master = mtd_get_master(mtd); + int npairs = mtd_wunit_per_eb(master) / mtd_pairing_groups(master); if (wunit < 0 || wunit >= npairs) return -EINVAL; - if (mtd->pairing && mtd->pairing->get_info) - return mtd->pairing->get_info(mtd, wunit, info); + if (master->pairing && master->pairing->get_info) + return master->pairing->get_info(master, wunit, info); info->group = 0; info->pair = wunit; @@ -498,15 +499,16 @@ EXPORT_SYMBOL_GPL(mtd_wunit_to_pairing_info); int mtd_pairing_info_to_wunit(struct mtd_info *mtd, const struct mtd_pairing_info *info) { - int ngroups = mtd_pairing_groups(mtd); - int npairs = mtd_wunit_per_eb(mtd) / ngroups; + struct mtd_info *master = mtd_get_master(mtd); + int ngroups = mtd_pairing_groups(master); + int npairs = mtd_wunit_per_eb(master) / ngroups; if (!info || info->pair < 0 || info->pair >= npairs || info->group < 0 || info->group >= ngroups) return -EINVAL; - if (mtd->pairing && mtd->pairing->get_wunit) - return mtd->pairing->get_wunit(mtd, info); + if (master->pairing && master->pairing->get_wunit) + return mtd->pairing->get_wunit(master, info); return info->pair; } @@ -524,10 +526,12 @@ EXPORT_SYMBOL_GPL(mtd_pairing_info_to_wunit); */ int mtd_pairing_groups(struct mtd_info *mtd) { - if (!mtd->pairing || !mtd->pairing->ngroups) + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->pairing || !master->pairing->ngroups) return 1; - return mtd->pairing->ngroups; + return master->pairing->ngroups; } EXPORT_SYMBOL_GPL(mtd_pairing_groups); @@ -587,6 +591,7 @@ static int mtd_nvmem_add(struct mtd_info *mtd) int add_mtd_device(struct mtd_info *mtd) { + struct mtd_info *master = mtd_get_master(mtd); struct mtd_notifier *not; int i, error; @@ -608,7 +613,7 @@ int add_mtd_device(struct mtd_info *mtd) (mtd->_read && mtd->_read_oob))) return -EINVAL; - if (WARN_ON((!mtd->erasesize || !mtd->_erase) && + if (WARN_ON((!mtd->erasesize || !master->_erase) && !(mtd->flags & MTD_NO_ERASE))) return -EINVAL; @@ -765,7 +770,8 @@ static void mtd_set_dev_defaults(struct mtd_info *mtd) pr_debug("mtd device won't show a device symlink in sysfs\n"); } - mtd->orig_flags = mtd->flags; + INIT_LIST_HEAD(&mtd->partitions); + mutex_init(&mtd->master.partitions_lock); } /** @@ -971,20 +977,26 @@ EXPORT_SYMBOL_GPL(get_mtd_device); int __get_mtd_device(struct mtd_info *mtd) { + struct mtd_info *master = mtd_get_master(mtd); int err; - if (!try_module_get(mtd->owner)) + if (!try_module_get(master->owner)) return -ENODEV; - if (mtd->_get_device) { - err = mtd->_get_device(mtd); + if (master->_get_device) { + err = master->_get_device(mtd); if (err) { - module_put(mtd->owner); + module_put(master->owner); return err; } } - mtd->usecount++; + + while (mtd->parent) { + mtd->usecount++; + mtd = mtd->parent; + } + return 0; } EXPORT_SYMBOL_GPL(__get_mtd_device); @@ -1038,13 +1050,18 @@ EXPORT_SYMBOL_GPL(put_mtd_device); void __put_mtd_device(struct mtd_info *mtd) { - --mtd->usecount; - BUG_ON(mtd->usecount < 0); + struct mtd_info *master = mtd_get_master(mtd); - if (mtd->_put_device) - mtd->_put_device(mtd); + while (mtd->parent) { + --mtd->usecount; + BUG_ON(mtd->usecount < 0); + mtd = mtd->parent; + } - module_put(mtd->owner); + if (master->_put_device) + master->_put_device(master); + + module_put(master->owner); } EXPORT_SYMBOL_GPL(__put_mtd_device); @@ -1055,9 +1072,13 @@ EXPORT_SYMBOL_GPL(__put_mtd_device); */ int mtd_erase(struct mtd_info *mtd, struct erase_info *instr) { + struct mtd_info *master = mtd_get_master(mtd); + u64 mst_ofs = mtd_get_master_ofs(mtd, 0); + int ret; + instr->fail_addr = MTD_FAIL_ADDR_UNKNOWN; - if (!mtd->erasesize || !mtd->_erase) + if (!mtd->erasesize || !master->_erase) return -ENOTSUPP; if (instr->addr >= mtd->size || instr->len > mtd->size - instr->addr) @@ -1069,7 +1090,14 @@ int mtd_erase(struct mtd_info *mtd, struct erase_info *instr) return 0; ledtrig_mtd_activity(); - return mtd->_erase(mtd, instr); + + instr->addr += mst_ofs; + ret = master->_erase(master, instr); + if (instr->fail_addr != MTD_FAIL_ADDR_UNKNOWN) + instr->fail_addr -= mst_ofs; + + instr->addr -= mst_ofs; + return ret; } EXPORT_SYMBOL_GPL(mtd_erase); @@ -1079,30 +1107,36 @@ EXPORT_SYMBOL_GPL(mtd_erase); int mtd_point(struct mtd_info *mtd, loff_t from, size_t len, size_t *retlen, void **virt, resource_size_t *phys) { + struct mtd_info *master = mtd_get_master(mtd); + *retlen = 0; *virt = NULL; if (phys) *phys = 0; - if (!mtd->_point) + if (!master->_point) return -EOPNOTSUPP; if (from < 0 || from >= mtd->size || len > mtd->size - from) return -EINVAL; if (!len) return 0; - return mtd->_point(mtd, from, len, retlen, virt, phys); + + from = mtd_get_master_ofs(mtd, from); + return master->_point(master, from, len, retlen, virt, phys); } EXPORT_SYMBOL_GPL(mtd_point); /* We probably shouldn't allow XIP if the unpoint isn't a NULL */ int mtd_unpoint(struct mtd_info *mtd, loff_t from, size_t len) { - if (!mtd->_unpoint) + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->_unpoint) return -EOPNOTSUPP; if (from < 0 || from >= mtd->size || len > mtd->size - from) return -EINVAL; if (!len) return 0; - return mtd->_unpoint(mtd, from, len); + return master->_unpoint(master, mtd_get_master_ofs(mtd, from), len); } EXPORT_SYMBOL_GPL(mtd_unpoint); @@ -1129,6 +1163,25 @@ unsigned long mtd_get_unmapped_area(struct mtd_info *mtd, unsigned long len, } EXPORT_SYMBOL_GPL(mtd_get_unmapped_area); +static void mtd_update_ecc_stats(struct mtd_info *mtd, struct mtd_info *master, + const struct mtd_ecc_stats *old_stats) +{ + struct mtd_ecc_stats diff; + + if (master == mtd) + return; + + diff = master->ecc_stats; + diff.failed -= old_stats->failed; + diff.corrected -= old_stats->corrected; + + while (mtd->parent) { + mtd->ecc_stats.failed += diff.failed; + mtd->ecc_stats.corrected += diff.corrected; + mtd = mtd->parent; + } +} + int mtd_read(struct mtd_info *mtd, loff_t from, size_t len, size_t *retlen, u_char *buf) { @@ -1171,8 +1224,10 @@ EXPORT_SYMBOL_GPL(mtd_write); int mtd_panic_write(struct mtd_info *mtd, loff_t to, size_t len, size_t *retlen, const u_char *buf) { + struct mtd_info *master = mtd_get_master(mtd); + *retlen = 0; - if (!mtd->_panic_write) + if (!master->_panic_write) return -EOPNOTSUPP; if (to < 0 || to >= mtd->size || len > mtd->size - to) return -EINVAL; @@ -1183,7 +1238,8 @@ int mtd_panic_write(struct mtd_info *mtd, loff_t to, size_t len, size_t *retlen, if (!mtd->oops_panic_write) mtd->oops_panic_write = true; - return mtd->_panic_write(mtd, to, len, retlen, buf); + return master->_panic_write(master, mtd_get_master_ofs(mtd, to), len, + retlen, buf); } EXPORT_SYMBOL_GPL(mtd_panic_write); @@ -1222,7 +1278,10 @@ static int mtd_check_oob_ops(struct mtd_info *mtd, loff_t offs, int mtd_read_oob(struct mtd_info *mtd, loff_t from, struct mtd_oob_ops *ops) { + struct mtd_info *master = mtd_get_master(mtd); + struct mtd_ecc_stats old_stats = master->ecc_stats; int ret_code; + ops->retlen = ops->oobretlen = 0; ret_code = mtd_check_oob_ops(mtd, from, ops); @@ -1232,14 +1291,17 @@ int mtd_read_oob(struct mtd_info *mtd, loff_t from, struct mtd_oob_ops *ops) ledtrig_mtd_activity(); /* Check the validity of a potential fallback on mtd->_read */ - if (!mtd->_read_oob && (!mtd->_read || ops->oobbuf)) + if (!master->_read_oob && (!master->_read || ops->oobbuf)) return -EOPNOTSUPP; - if (mtd->_read_oob) - ret_code = mtd->_read_oob(mtd, from, ops); + from = mtd_get_master_ofs(mtd, from); + if (master->_read_oob) + ret_code = master->_read_oob(master, from, ops); else - ret_code = mtd->_read(mtd, from, ops->len, &ops->retlen, - ops->datbuf); + ret_code = master->_read(master, from, ops->len, &ops->retlen, + ops->datbuf); + + mtd_update_ecc_stats(mtd, master, &old_stats); /* * In cases where ops->datbuf != NULL, mtd->_read_oob() has semantics @@ -1258,6 +1320,7 @@ EXPORT_SYMBOL_GPL(mtd_read_oob); int mtd_write_oob(struct mtd_info *mtd, loff_t to, struct mtd_oob_ops *ops) { + struct mtd_info *master = mtd_get_master(mtd); int ret; ops->retlen = ops->oobretlen = 0; @@ -1272,14 +1335,16 @@ int mtd_write_oob(struct mtd_info *mtd, loff_t to, ledtrig_mtd_activity(); /* Check the validity of a potential fallback on mtd->_write */ - if (!mtd->_write_oob && (!mtd->_write || ops->oobbuf)) + if (!master->_write_oob && (!master->_write || ops->oobbuf)) return -EOPNOTSUPP; - if (mtd->_write_oob) - return mtd->_write_oob(mtd, to, ops); + to = mtd_get_master_ofs(mtd, to); + + if (master->_write_oob) + return master->_write_oob(master, to, ops); else - return mtd->_write(mtd, to, ops->len, &ops->retlen, - ops->datbuf); + return master->_write(master, to, ops->len, &ops->retlen, + ops->datbuf); } EXPORT_SYMBOL_GPL(mtd_write_oob); @@ -1302,15 +1367,17 @@ EXPORT_SYMBOL_GPL(mtd_write_oob); int mtd_ooblayout_ecc(struct mtd_info *mtd, int section, struct mtd_oob_region *oobecc) { + struct mtd_info *master = mtd_get_master(mtd); + memset(oobecc, 0, sizeof(*oobecc)); - if (!mtd || section < 0) + if (!master || section < 0) return -EINVAL; - if (!mtd->ooblayout || !mtd->ooblayout->ecc) + if (!master->ooblayout || !master->ooblayout->ecc) return -ENOTSUPP; - return mtd->ooblayout->ecc(mtd, section, oobecc); + return master->ooblayout->ecc(master, section, oobecc); } EXPORT_SYMBOL_GPL(mtd_ooblayout_ecc); @@ -1334,15 +1401,17 @@ EXPORT_SYMBOL_GPL(mtd_ooblayout_ecc); int mtd_ooblayout_free(struct mtd_info *mtd, int section, struct mtd_oob_region *oobfree) { + struct mtd_info *master = mtd_get_master(mtd); + memset(oobfree, 0, sizeof(*oobfree)); - if (!mtd || section < 0) + if (!master || section < 0) return -EINVAL; - if (!mtd->ooblayout || !mtd->ooblayout->free) + if (!master->ooblayout || !master->ooblayout->free) return -ENOTSUPP; - return mtd->ooblayout->free(mtd, section, oobfree); + return master->ooblayout->free(master, section, oobfree); } EXPORT_SYMBOL_GPL(mtd_ooblayout_free); @@ -1651,60 +1720,69 @@ EXPORT_SYMBOL_GPL(mtd_ooblayout_count_eccbytes); int mtd_get_fact_prot_info(struct mtd_info *mtd, size_t len, size_t *retlen, struct otp_info *buf) { - if (!mtd->_get_fact_prot_info) + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->_get_fact_prot_info) return -EOPNOTSUPP; if (!len) return 0; - return mtd->_get_fact_prot_info(mtd, len, retlen, buf); + return master->_get_fact_prot_info(master, len, retlen, buf); } EXPORT_SYMBOL_GPL(mtd_get_fact_prot_info); int mtd_read_fact_prot_reg(struct mtd_info *mtd, loff_t from, size_t len, size_t *retlen, u_char *buf) { + struct mtd_info *master = mtd_get_master(mtd); + *retlen = 0; - if (!mtd->_read_fact_prot_reg) + if (!master->_read_fact_prot_reg) return -EOPNOTSUPP; if (!len) return 0; - return mtd->_read_fact_prot_reg(mtd, from, len, retlen, buf); + return master->_read_fact_prot_reg(master, from, len, retlen, buf); } EXPORT_SYMBOL_GPL(mtd_read_fact_prot_reg); int mtd_get_user_prot_info(struct mtd_info *mtd, size_t len, size_t *retlen, struct otp_info *buf) { - if (!mtd->_get_user_prot_info) + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->_get_user_prot_info) return -EOPNOTSUPP; if (!len) return 0; - return mtd->_get_user_prot_info(mtd, len, retlen, buf); + return master->_get_user_prot_info(master, len, retlen, buf); } EXPORT_SYMBOL_GPL(mtd_get_user_prot_info); int mtd_read_user_prot_reg(struct mtd_info *mtd, loff_t from, size_t len, size_t *retlen, u_char *buf) { + struct mtd_info *master = mtd_get_master(mtd); + *retlen = 0; - if (!mtd->_read_user_prot_reg) + if (!master->_read_user_prot_reg) return -EOPNOTSUPP; if (!len) return 0; - return mtd->_read_user_prot_reg(mtd, from, len, retlen, buf); + return master->_read_user_prot_reg(master, from, len, retlen, buf); } EXPORT_SYMBOL_GPL(mtd_read_user_prot_reg); int mtd_write_user_prot_reg(struct mtd_info *mtd, loff_t to, size_t len, size_t *retlen, u_char *buf) { + struct mtd_info *master = mtd_get_master(mtd); int ret; *retlen = 0; - if (!mtd->_write_user_prot_reg) + if (!master->_write_user_prot_reg) return -EOPNOTSUPP; if (!len) return 0; - ret = mtd->_write_user_prot_reg(mtd, to, len, retlen, buf); + ret = master->_write_user_prot_reg(master, to, len, retlen, buf); if (ret) return ret; @@ -1718,80 +1796,105 @@ EXPORT_SYMBOL_GPL(mtd_write_user_prot_reg); int mtd_lock_user_prot_reg(struct mtd_info *mtd, loff_t from, size_t len) { - if (!mtd->_lock_user_prot_reg) + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->_lock_user_prot_reg) return -EOPNOTSUPP; if (!len) return 0; - return mtd->_lock_user_prot_reg(mtd, from, len); + return master->_lock_user_prot_reg(master, from, len); } EXPORT_SYMBOL_GPL(mtd_lock_user_prot_reg); /* Chip-supported device locking */ int mtd_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) { - if (!mtd->_lock) + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->_lock) return -EOPNOTSUPP; if (ofs < 0 || ofs >= mtd->size || len > mtd->size - ofs) return -EINVAL; if (!len) return 0; - return mtd->_lock(mtd, ofs, len); + return master->_lock(master, mtd_get_master_ofs(mtd, ofs), len); } EXPORT_SYMBOL_GPL(mtd_lock); int mtd_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) { - if (!mtd->_unlock) + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->_unlock) return -EOPNOTSUPP; if (ofs < 0 || ofs >= mtd->size || len > mtd->size - ofs) return -EINVAL; if (!len) return 0; - return mtd->_unlock(mtd, ofs, len); + return master->_unlock(master, mtd_get_master_ofs(mtd, ofs), len); } EXPORT_SYMBOL_GPL(mtd_unlock); int mtd_is_locked(struct mtd_info *mtd, loff_t ofs, uint64_t len) { - if (!mtd->_is_locked) + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->_is_locked) return -EOPNOTSUPP; if (ofs < 0 || ofs >= mtd->size || len > mtd->size - ofs) return -EINVAL; if (!len) return 0; - return mtd->_is_locked(mtd, ofs, len); + return master->_is_locked(master, mtd_get_master_ofs(mtd, ofs), len); } EXPORT_SYMBOL_GPL(mtd_is_locked); int mtd_block_isreserved(struct mtd_info *mtd, loff_t ofs) { + struct mtd_info *master = mtd_get_master(mtd); + if (ofs < 0 || ofs >= mtd->size) return -EINVAL; - if (!mtd->_block_isreserved) + if (!master->_block_isreserved) return 0; - return mtd->_block_isreserved(mtd, ofs); + return master->_block_isreserved(master, mtd_get_master_ofs(mtd, ofs)); } EXPORT_SYMBOL_GPL(mtd_block_isreserved); int mtd_block_isbad(struct mtd_info *mtd, loff_t ofs) { + struct mtd_info *master = mtd_get_master(mtd); + if (ofs < 0 || ofs >= mtd->size) return -EINVAL; - if (!mtd->_block_isbad) + if (!master->_block_isbad) return 0; - return mtd->_block_isbad(mtd, ofs); + return master->_block_isbad(master, mtd_get_master_ofs(mtd, ofs)); } EXPORT_SYMBOL_GPL(mtd_block_isbad); int mtd_block_markbad(struct mtd_info *mtd, loff_t ofs) { - if (!mtd->_block_markbad) + struct mtd_info *master = mtd_get_master(mtd); + int ret; + + if (!master->_block_markbad) return -EOPNOTSUPP; if (ofs < 0 || ofs >= mtd->size) return -EINVAL; if (!(mtd->flags & MTD_WRITEABLE)) return -EROFS; - return mtd->_block_markbad(mtd, ofs); + + ret = master->_block_markbad(master, mtd_get_master_ofs(mtd, ofs)); + if (ret) + return ret; + + while (mtd->parent) { + mtd->ecc_stats.badblocks++; + mtd = mtd->parent; + } + + return 0; } EXPORT_SYMBOL_GPL(mtd_block_markbad); @@ -1841,12 +1944,17 @@ static int default_mtd_writev(struct mtd_info *mtd, const struct kvec *vecs, int mtd_writev(struct mtd_info *mtd, const struct kvec *vecs, unsigned long count, loff_t to, size_t *retlen) { + struct mtd_info *master = mtd_get_master(mtd); + *retlen = 0; if (!(mtd->flags & MTD_WRITEABLE)) return -EROFS; - if (!mtd->_writev) + + if (!master->_writev) return default_mtd_writev(mtd, vecs, count, to, retlen); - return mtd->_writev(mtd, vecs, count, to, retlen); + + return master->_writev(master, vecs, count, + mtd_get_master_ofs(mtd, to), retlen); } EXPORT_SYMBOL_GPL(mtd_writev); diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c index 7328c066c5ba..3f6025684f58 100644 --- a/drivers/mtd/mtdpart.c +++ b/drivers/mtd/mtdpart.c @@ -20,339 +20,52 @@ #include "mtdcore.h" -/* Our partition linked list */ -static LIST_HEAD(mtd_partitions); -static DEFINE_MUTEX(mtd_partitions_mutex); - -/** - * struct mtd_part - our partition node structure - * - * @mtd: struct holding partition details - * @parent: parent mtd - flash device or another partition - * @offset: partition offset relative to the *flash device* - */ -struct mtd_part { - struct mtd_info mtd; - struct mtd_info *parent; - uint64_t offset; - struct list_head list; -}; - -/* - * Given a pointer to the MTD object in the mtd_part structure, we can retrieve - * the pointer to that structure. - */ -static inline struct mtd_part *mtd_to_part(const struct mtd_info *mtd) -{ - return container_of(mtd, struct mtd_part, mtd); -} - -static u64 part_absolute_offset(struct mtd_info *mtd) -{ - struct mtd_part *part = mtd_to_part(mtd); - - if (!mtd_is_partition(mtd)) - return 0; - - return part_absolute_offset(part->parent) + part->offset; -} - /* * MTD methods which simply translate the effective address and pass through * to the _real_ device. */ -static int part_read(struct mtd_info *mtd, loff_t from, size_t len, - size_t *retlen, u_char *buf) +static inline void free_partition(struct mtd_info *mtd) { - struct mtd_part *part = mtd_to_part(mtd); - struct mtd_ecc_stats stats; - int res; - - stats = part->parent->ecc_stats; - res = part->parent->_read(part->parent, from + part->offset, len, - retlen, buf); - if (unlikely(mtd_is_eccerr(res))) - mtd->ecc_stats.failed += - part->parent->ecc_stats.failed - stats.failed; - else - mtd->ecc_stats.corrected += - part->parent->ecc_stats.corrected - stats.corrected; - return res; + kfree(mtd->name); + kfree(mtd); } -static int part_point(struct mtd_info *mtd, loff_t from, size_t len, - size_t *retlen, void **virt, resource_size_t *phys) -{ - struct mtd_part *part = mtd_to_part(mtd); - - return part->parent->_point(part->parent, from + part->offset, len, - retlen, virt, phys); -} - -static int part_unpoint(struct mtd_info *mtd, loff_t from, size_t len) -{ - struct mtd_part *part = mtd_to_part(mtd); - - return part->parent->_unpoint(part->parent, from + part->offset, len); -} - -static int part_read_oob(struct mtd_info *mtd, loff_t from, - struct mtd_oob_ops *ops) -{ - struct mtd_part *part = mtd_to_part(mtd); - struct mtd_ecc_stats stats; - int res; - - stats = part->parent->ecc_stats; - res = part->parent->_read_oob(part->parent, from + part->offset, ops); - if (unlikely(mtd_is_eccerr(res))) - mtd->ecc_stats.failed += - part->parent->ecc_stats.failed - stats.failed; - else - mtd->ecc_stats.corrected += - part->parent->ecc_stats.corrected - stats.corrected; - return res; -} - -static int part_read_user_prot_reg(struct mtd_info *mtd, loff_t from, - size_t len, size_t *retlen, u_char *buf) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_read_user_prot_reg(part->parent, from, len, - retlen, buf); -} - -static int part_get_user_prot_info(struct mtd_info *mtd, size_t len, - size_t *retlen, struct otp_info *buf) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_get_user_prot_info(part->parent, len, retlen, - buf); -} - -static int part_read_fact_prot_reg(struct mtd_info *mtd, loff_t from, - size_t len, size_t *retlen, u_char *buf) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_read_fact_prot_reg(part->parent, from, len, - retlen, buf); -} - -static int part_get_fact_prot_info(struct mtd_info *mtd, size_t len, - size_t *retlen, struct otp_info *buf) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_get_fact_prot_info(part->parent, len, retlen, - buf); -} - -static int part_write(struct mtd_info *mtd, loff_t to, size_t len, - size_t *retlen, const u_char *buf) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_write(part->parent, to + part->offset, len, - retlen, buf); -} - -static int part_panic_write(struct mtd_info *mtd, loff_t to, size_t len, - size_t *retlen, const u_char *buf) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_panic_write(part->parent, to + part->offset, len, - retlen, buf); -} - -static int part_write_oob(struct mtd_info *mtd, loff_t to, - struct mtd_oob_ops *ops) -{ - struct mtd_part *part = mtd_to_part(mtd); - - return part->parent->_write_oob(part->parent, to + part->offset, ops); -} - -static int part_write_user_prot_reg(struct mtd_info *mtd, loff_t from, - size_t len, size_t *retlen, u_char *buf) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_write_user_prot_reg(part->parent, from, len, - retlen, buf); -} - -static int part_lock_user_prot_reg(struct mtd_info *mtd, loff_t from, - size_t len) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_lock_user_prot_reg(part->parent, from, len); -} - -static int part_writev(struct mtd_info *mtd, const struct kvec *vecs, - unsigned long count, loff_t to, size_t *retlen) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_writev(part->parent, vecs, count, - to + part->offset, retlen); -} - -static int part_erase(struct mtd_info *mtd, struct erase_info *instr) -{ - struct mtd_part *part = mtd_to_part(mtd); - int ret; - - instr->addr += part->offset; - ret = part->parent->_erase(part->parent, instr); - if (instr->fail_addr != MTD_FAIL_ADDR_UNKNOWN) - instr->fail_addr -= part->offset; - instr->addr -= part->offset; - - return ret; -} - -static int part_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_lock(part->parent, ofs + part->offset, len); -} - -static int part_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_unlock(part->parent, ofs + part->offset, len); -} - -static int part_is_locked(struct mtd_info *mtd, loff_t ofs, uint64_t len) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_is_locked(part->parent, ofs + part->offset, len); -} - -static void part_sync(struct mtd_info *mtd) -{ - struct mtd_part *part = mtd_to_part(mtd); - part->parent->_sync(part->parent); -} - -static int part_suspend(struct mtd_info *mtd) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_suspend(part->parent); -} - -static void part_resume(struct mtd_info *mtd) -{ - struct mtd_part *part = mtd_to_part(mtd); - part->parent->_resume(part->parent); -} - -static int part_block_isreserved(struct mtd_info *mtd, loff_t ofs) -{ - struct mtd_part *part = mtd_to_part(mtd); - ofs += part->offset; - return part->parent->_block_isreserved(part->parent, ofs); -} - -static int part_block_isbad(struct mtd_info *mtd, loff_t ofs) -{ - struct mtd_part *part = mtd_to_part(mtd); - ofs += part->offset; - return part->parent->_block_isbad(part->parent, ofs); -} - -static int part_block_markbad(struct mtd_info *mtd, loff_t ofs) -{ - struct mtd_part *part = mtd_to_part(mtd); - int res; - - ofs += part->offset; - res = part->parent->_block_markbad(part->parent, ofs); - if (!res) - mtd->ecc_stats.badblocks++; - return res; -} - -static int part_get_device(struct mtd_info *mtd) -{ - struct mtd_part *part = mtd_to_part(mtd); - return part->parent->_get_device(part->parent); -} - -static void part_put_device(struct mtd_info *mtd) -{ - struct mtd_part *part = mtd_to_part(mtd); - part->parent->_put_device(part->parent); -} - -static int part_ooblayout_ecc(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct mtd_part *part = mtd_to_part(mtd); - - return mtd_ooblayout_ecc(part->parent, section, oobregion); -} - -static int part_ooblayout_free(struct mtd_info *mtd, int section, - struct mtd_oob_region *oobregion) -{ - struct mtd_part *part = mtd_to_part(mtd); - - return mtd_ooblayout_free(part->parent, section, oobregion); -} - -static const struct mtd_ooblayout_ops part_ooblayout_ops = { - .ecc = part_ooblayout_ecc, - .free = part_ooblayout_free, -}; - -static int part_max_bad_blocks(struct mtd_info *mtd, loff_t ofs, size_t len) -{ - struct mtd_part *part = mtd_to_part(mtd); - - return part->parent->_max_bad_blocks(part->parent, - ofs + part->offset, len); -} - -static inline void free_partition(struct mtd_part *p) -{ - kfree(p->mtd.name); - kfree(p); -} - -static struct mtd_part *allocate_partition(struct mtd_info *parent, - const struct mtd_partition *part, int partno, - uint64_t cur_offset) +static struct mtd_info *allocate_partition(struct mtd_info *parent, + const struct mtd_partition *part, + int partno, uint64_t cur_offset) { int wr_alignment = (parent->flags & MTD_NO_ERASE) ? parent->writesize : parent->erasesize; - struct mtd_part *slave; + struct mtd_info *child, *master = mtd_get_master(parent); u32 remainder; char *name; u64 tmp; /* allocate the partition structure */ - slave = kzalloc(sizeof(*slave), GFP_KERNEL); + child = kzalloc(sizeof(*child), GFP_KERNEL); name = kstrdup(part->name, GFP_KERNEL); - if (!name || !slave) { + if (!name || !child) { printk(KERN_ERR"memory allocation error while creating partitions for \"%s\"\n", parent->name); kfree(name); - kfree(slave); + kfree(child); return ERR_PTR(-ENOMEM); } /* set up the MTD object for this partition */ - slave->mtd.type = parent->type; - slave->mtd.flags = parent->orig_flags & ~part->mask_flags; - slave->mtd.orig_flags = slave->mtd.flags; - slave->mtd.size = part->size; - slave->mtd.writesize = parent->writesize; - slave->mtd.writebufsize = parent->writebufsize; - slave->mtd.oobsize = parent->oobsize; - slave->mtd.oobavail = parent->oobavail; - slave->mtd.subpage_sft = parent->subpage_sft; - slave->mtd.pairing = parent->pairing; + child->type = parent->type; + child->part.flags = parent->flags & ~part->mask_flags; + child->flags = child->part.flags; + child->size = part->size; + child->writesize = parent->writesize; + child->writebufsize = parent->writebufsize; + child->oobsize = parent->oobsize; + child->oobavail = parent->oobavail; + child->subpage_sft = parent->subpage_sft; - slave->mtd.name = name; - slave->mtd.owner = parent->owner; + child->name = name; + child->owner = parent->owner; /* NOTE: Historically, we didn't arrange MTDs as a tree out of * concern for showing the same data in multiple partitions. @@ -360,134 +73,76 @@ static struct mtd_part *allocate_partition(struct mtd_info *parent, * so the MTD_PARTITIONED_MASTER option allows that. The master * will have device nodes etc only if this is set, so make the * parent conditional on that option. Note, this is a way to - * distinguish between the master and the partition in sysfs. + * distinguish between the parent and its partitions in sysfs. */ - slave->mtd.dev.parent = IS_ENABLED(CONFIG_MTD_PARTITIONED_MASTER) || mtd_is_partition(parent) ? - &parent->dev : - parent->dev.parent; - slave->mtd.dev.of_node = part->of_node; + child->dev.parent = IS_ENABLED(CONFIG_MTD_PARTITIONED_MASTER) || mtd_is_partition(parent) ? + &parent->dev : parent->dev.parent; + child->dev.of_node = part->of_node; + child->parent = parent; + child->part.offset = part->offset; + INIT_LIST_HEAD(&child->partitions); - if (parent->_read) - slave->mtd._read = part_read; - if (parent->_write) - slave->mtd._write = part_write; - - if (parent->_panic_write) - slave->mtd._panic_write = part_panic_write; - - if (parent->_point && parent->_unpoint) { - slave->mtd._point = part_point; - slave->mtd._unpoint = part_unpoint; - } - - if (parent->_read_oob) - slave->mtd._read_oob = part_read_oob; - if (parent->_write_oob) - slave->mtd._write_oob = part_write_oob; - if (parent->_read_user_prot_reg) - slave->mtd._read_user_prot_reg = part_read_user_prot_reg; - if (parent->_read_fact_prot_reg) - slave->mtd._read_fact_prot_reg = part_read_fact_prot_reg; - if (parent->_write_user_prot_reg) - slave->mtd._write_user_prot_reg = part_write_user_prot_reg; - if (parent->_lock_user_prot_reg) - slave->mtd._lock_user_prot_reg = part_lock_user_prot_reg; - if (parent->_get_user_prot_info) - slave->mtd._get_user_prot_info = part_get_user_prot_info; - if (parent->_get_fact_prot_info) - slave->mtd._get_fact_prot_info = part_get_fact_prot_info; - if (parent->_sync) - slave->mtd._sync = part_sync; - if (!partno && !parent->dev.class && parent->_suspend && - parent->_resume) { - slave->mtd._suspend = part_suspend; - slave->mtd._resume = part_resume; - } - if (parent->_writev) - slave->mtd._writev = part_writev; - if (parent->_lock) - slave->mtd._lock = part_lock; - if (parent->_unlock) - slave->mtd._unlock = part_unlock; - if (parent->_is_locked) - slave->mtd._is_locked = part_is_locked; - if (parent->_block_isreserved) - slave->mtd._block_isreserved = part_block_isreserved; - if (parent->_block_isbad) - slave->mtd._block_isbad = part_block_isbad; - if (parent->_block_markbad) - slave->mtd._block_markbad = part_block_markbad; - if (parent->_max_bad_blocks) - slave->mtd._max_bad_blocks = part_max_bad_blocks; - - if (parent->_get_device) - slave->mtd._get_device = part_get_device; - if (parent->_put_device) - slave->mtd._put_device = part_put_device; - - slave->mtd._erase = part_erase; - slave->parent = parent; - slave->offset = part->offset; - - if (slave->offset == MTDPART_OFS_APPEND) - slave->offset = cur_offset; - if (slave->offset == MTDPART_OFS_NXTBLK) { + if (child->part.offset == MTDPART_OFS_APPEND) + child->part.offset = cur_offset; + if (child->part.offset == MTDPART_OFS_NXTBLK) { tmp = cur_offset; - slave->offset = cur_offset; + child->part.offset = cur_offset; remainder = do_div(tmp, wr_alignment); if (remainder) { - slave->offset += wr_alignment - remainder; + child->part.offset += wr_alignment - remainder; printk(KERN_NOTICE "Moving partition %d: " "0x%012llx -> 0x%012llx\n", partno, - (unsigned long long)cur_offset, (unsigned long long)slave->offset); + (unsigned long long)cur_offset, + child->part.offset); } } - if (slave->offset == MTDPART_OFS_RETAIN) { - slave->offset = cur_offset; - if (parent->size - slave->offset >= slave->mtd.size) { - slave->mtd.size = parent->size - slave->offset - - slave->mtd.size; + if (child->part.offset == MTDPART_OFS_RETAIN) { + child->part.offset = cur_offset; + if (parent->size - child->part.offset >= child->size) { + child->size = parent->size - child->part.offset - + child->size; } else { printk(KERN_ERR "mtd partition \"%s\" doesn't have enough space: %#llx < %#llx, disabled\n", - part->name, parent->size - slave->offset, - slave->mtd.size); + part->name, parent->size - child->part.offset, + child->size); /* register to preserve ordering */ goto out_register; } } - if (slave->mtd.size == MTDPART_SIZ_FULL) - slave->mtd.size = parent->size - slave->offset; + if (child->size == MTDPART_SIZ_FULL) + child->size = parent->size - child->part.offset; - printk(KERN_NOTICE "0x%012llx-0x%012llx : \"%s\"\n", (unsigned long long)slave->offset, - (unsigned long long)(slave->offset + slave->mtd.size), slave->mtd.name); + printk(KERN_NOTICE "0x%012llx-0x%012llx : \"%s\"\n", + child->part.offset, child->part.offset + child->size, + child->name); /* let's do some sanity checks */ - if (slave->offset >= parent->size) { + if (child->part.offset >= parent->size) { /* let's register it anyway to preserve ordering */ - slave->offset = 0; - slave->mtd.size = 0; + child->part.offset = 0; + child->size = 0; /* Initialize ->erasesize to make add_mtd_device() happy. */ - slave->mtd.erasesize = parent->erasesize; - + child->erasesize = parent->erasesize; printk(KERN_ERR"mtd: partition \"%s\" is out of reach -- disabled\n", part->name); goto out_register; } - if (slave->offset + slave->mtd.size > parent->size) { - slave->mtd.size = parent->size - slave->offset; + if (child->part.offset + child->size > parent->size) { + child->size = parent->size - child->part.offset; printk(KERN_WARNING"mtd: partition \"%s\" extends beyond the end of device \"%s\" -- size truncated to %#llx\n", - part->name, parent->name, (unsigned long long)slave->mtd.size); + part->name, parent->name, child->size); } if (parent->numeraseregions > 1) { /* Deal with variable erase size stuff */ int i, max = parent->numeraseregions; - u64 end = slave->offset + slave->mtd.size; + u64 end = child->part.offset + child->size; struct mtd_erase_region_info *regions = parent->eraseregions; /* Find the first erase regions which is part of this * partition. */ - for (i = 0; i < max && regions[i].offset <= slave->offset; i++) + for (i = 0; i < max && regions[i].offset <= child->part.offset; + i++) ; /* The loop searched for the region _behind_ the first one */ if (i > 0) @@ -495,70 +150,68 @@ static struct mtd_part *allocate_partition(struct mtd_info *parent, /* Pick biggest erasesize */ for (; i < max && regions[i].offset < end; i++) { - if (slave->mtd.erasesize < regions[i].erasesize) { - slave->mtd.erasesize = regions[i].erasesize; - } + if (child->erasesize < regions[i].erasesize) + child->erasesize = regions[i].erasesize; } - BUG_ON(slave->mtd.erasesize == 0); + BUG_ON(child->erasesize == 0); } else { /* Single erase size */ - slave->mtd.erasesize = parent->erasesize; + child->erasesize = parent->erasesize; } /* - * Slave erasesize might differ from the master one if the master + * Child erasesize might differ from the parent one if the parent * exposes several regions with different erasesize. Adjust * wr_alignment accordingly. */ - if (!(slave->mtd.flags & MTD_NO_ERASE)) - wr_alignment = slave->mtd.erasesize; + if (!(child->flags & MTD_NO_ERASE)) + wr_alignment = child->erasesize; - tmp = part_absolute_offset(parent) + slave->offset; + tmp = mtd_get_master_ofs(child, 0); remainder = do_div(tmp, wr_alignment); - if ((slave->mtd.flags & MTD_WRITEABLE) && remainder) { + if ((child->flags & MTD_WRITEABLE) && remainder) { /* Doesn't start on a boundary of major erase size */ /* FIXME: Let it be writable if it is on a boundary of * _minor_ erase size though */ - slave->mtd.flags &= ~MTD_WRITEABLE; + child->flags &= ~MTD_WRITEABLE; printk(KERN_WARNING"mtd: partition \"%s\" doesn't start on an erase/write block boundary -- force read-only\n", part->name); } - tmp = part_absolute_offset(parent) + slave->mtd.size; + tmp = mtd_get_master_ofs(child, 0) + child->size; remainder = do_div(tmp, wr_alignment); - if ((slave->mtd.flags & MTD_WRITEABLE) && remainder) { - slave->mtd.flags &= ~MTD_WRITEABLE; + if ((child->flags & MTD_WRITEABLE) && remainder) { + child->flags &= ~MTD_WRITEABLE; printk(KERN_WARNING"mtd: partition \"%s\" doesn't end on an erase/write block -- force read-only\n", part->name); } - mtd_set_ooblayout(&slave->mtd, &part_ooblayout_ops); - slave->mtd.ecc_step_size = parent->ecc_step_size; - slave->mtd.ecc_strength = parent->ecc_strength; - slave->mtd.bitflip_threshold = parent->bitflip_threshold; + child->ecc_step_size = parent->ecc_step_size; + child->ecc_strength = parent->ecc_strength; + child->bitflip_threshold = parent->bitflip_threshold; - if (parent->_block_isbad) { + if (master->_block_isbad) { uint64_t offs = 0; - while (offs < slave->mtd.size) { - if (mtd_block_isreserved(parent, offs + slave->offset)) - slave->mtd.ecc_stats.bbtblocks++; - else if (mtd_block_isbad(parent, offs + slave->offset)) - slave->mtd.ecc_stats.badblocks++; - offs += slave->mtd.erasesize; + while (offs < child->size) { + if (mtd_block_isreserved(child, offs)) + child->ecc_stats.bbtblocks++; + else if (mtd_block_isbad(child, offs)) + child->ecc_stats.badblocks++; + offs += child->erasesize; } } out_register: - return slave; + return child; } static ssize_t mtd_partition_offset_show(struct device *dev, struct device_attribute *attr, char *buf) { struct mtd_info *mtd = dev_get_drvdata(dev); - struct mtd_part *part = mtd_to_part(mtd); - return snprintf(buf, PAGE_SIZE, "%llu\n", part->offset); + + return snprintf(buf, PAGE_SIZE, "%lld\n", mtd->part.offset); } static DEVICE_ATTR(offset, S_IRUGO, mtd_partition_offset_show, NULL); @@ -568,9 +221,9 @@ static const struct attribute *mtd_partition_attrs[] = { NULL }; -static int mtd_add_partition_attrs(struct mtd_part *new) +static int mtd_add_partition_attrs(struct mtd_info *new) { - int ret = sysfs_create_files(&new->mtd.dev.kobj, mtd_partition_attrs); + int ret = sysfs_create_files(&new->dev.kobj, mtd_partition_attrs); if (ret) printk(KERN_WARNING "mtd: failed to create partition attrs, err=%d\n", ret); @@ -580,8 +233,9 @@ static int mtd_add_partition_attrs(struct mtd_part *new) int mtd_add_partition(struct mtd_info *parent, const char *name, long long offset, long long length) { + struct mtd_info *master = mtd_get_master(parent); struct mtd_partition part; - struct mtd_part *new; + struct mtd_info *child; int ret = 0; /* the direct offset is expected */ @@ -600,28 +254,28 @@ int mtd_add_partition(struct mtd_info *parent, const char *name, part.size = length; part.offset = offset; - new = allocate_partition(parent, &part, -1, offset); - if (IS_ERR(new)) - return PTR_ERR(new); + child = allocate_partition(parent, &part, -1, offset); + if (IS_ERR(child)) + return PTR_ERR(child); - mutex_lock(&mtd_partitions_mutex); - list_add(&new->list, &mtd_partitions); - mutex_unlock(&mtd_partitions_mutex); + mutex_lock(&master->master.partitions_lock); + list_add_tail(&child->part.node, &parent->partitions); + mutex_unlock(&master->master.partitions_lock); - ret = add_mtd_device(&new->mtd); + ret = add_mtd_device(child); if (ret) goto err_remove_part; - mtd_add_partition_attrs(new); + mtd_add_partition_attrs(child); return 0; err_remove_part: - mutex_lock(&mtd_partitions_mutex); - list_del(&new->list); - mutex_unlock(&mtd_partitions_mutex); + mutex_lock(&master->master.partitions_lock); + list_del(&child->part.node); + mutex_unlock(&master->master.partitions_lock); - free_partition(new); + free_partition(child); return ret; } @@ -630,119 +284,142 @@ EXPORT_SYMBOL_GPL(mtd_add_partition); /** * __mtd_del_partition - delete MTD partition * - * @priv: internal MTD struct for partition to be deleted + * @priv: MTD structure to be deleted * * This function must be called with the partitions mutex locked. */ -static int __mtd_del_partition(struct mtd_part *priv) +static int __mtd_del_partition(struct mtd_info *mtd) { - struct mtd_part *child, *next; + struct mtd_info *child, *next; int err; - list_for_each_entry_safe(child, next, &mtd_partitions, list) { - if (child->parent == &priv->mtd) { - err = __mtd_del_partition(child); - if (err) - return err; - } + list_for_each_entry_safe(child, next, &mtd->partitions, part.node) { + err = __mtd_del_partition(child); + if (err) + return err; } - sysfs_remove_files(&priv->mtd.dev.kobj, mtd_partition_attrs); + sysfs_remove_files(&mtd->dev.kobj, mtd_partition_attrs); - err = del_mtd_device(&priv->mtd); + err = del_mtd_device(mtd); if (err) return err; - list_del(&priv->list); - free_partition(priv); + list_del(&child->part.node); + free_partition(mtd); return 0; } /* * This function unregisters and destroy all slave MTD objects which are - * attached to the given MTD object. + * attached to the given MTD object, recursively. */ -int del_mtd_partitions(struct mtd_info *mtd) +static int __del_mtd_partitions(struct mtd_info *mtd) { - struct mtd_part *slave, *next; + struct mtd_info *child, *next; + LIST_HEAD(tmp_list); int ret, err = 0; - mutex_lock(&mtd_partitions_mutex); - list_for_each_entry_safe(slave, next, &mtd_partitions, list) - if (slave->parent == mtd) { - ret = __mtd_del_partition(slave); - if (ret < 0) - err = ret; + list_for_each_entry_safe(child, next, &mtd->partitions, part.node) { + if (mtd_has_partitions(child)) + del_mtd_partitions(child); + + pr_info("Deleting %s MTD partition\n", child->name); + ret = del_mtd_device(child); + if (ret < 0) { + pr_err("Error when deleting partition \"%s\" (%d)\n", + child->name, ret); + err = ret; + continue; } - mutex_unlock(&mtd_partitions_mutex); + + list_del(&child->part.node); + free_partition(child); + } return err; } +int del_mtd_partitions(struct mtd_info *mtd) +{ + struct mtd_info *master = mtd_get_master(mtd); + int ret; + + pr_info("Deleting MTD partitions on \"%s\":\n", mtd->name); + + mutex_lock(&master->master.partitions_lock); + ret = __del_mtd_partitions(mtd); + mutex_unlock(&master->master.partitions_lock); + + return ret; +} + int mtd_del_partition(struct mtd_info *mtd, int partno) { - struct mtd_part *slave, *next; + struct mtd_info *child, *master = mtd_get_master(mtd); int ret = -EINVAL; - mutex_lock(&mtd_partitions_mutex); - list_for_each_entry_safe(slave, next, &mtd_partitions, list) - if ((slave->parent == mtd) && - (slave->mtd.index == partno)) { - ret = __mtd_del_partition(slave); + mutex_lock(&master->master.partitions_lock); + list_for_each_entry(child, &mtd->partitions, part.node) { + if (child->index == partno) { + ret = __mtd_del_partition(child); break; } - mutex_unlock(&mtd_partitions_mutex); + } + mutex_unlock(&master->master.partitions_lock); return ret; } EXPORT_SYMBOL_GPL(mtd_del_partition); /* - * This function, given a master MTD object and a partition table, creates - * and registers slave MTD objects which are bound to the master according to - * the partition definitions. + * This function, given a parent MTD object and a partition table, creates + * and registers the child MTD objects which are bound to the parent according + * to the partition definitions. * - * For historical reasons, this function's caller only registers the master + * For historical reasons, this function's caller only registers the parent * if the MTD_PARTITIONED_MASTER config option is set. */ -int add_mtd_partitions(struct mtd_info *master, +int add_mtd_partitions(struct mtd_info *parent, const struct mtd_partition *parts, int nbparts) { - struct mtd_part *slave; + struct mtd_info *child, *master = mtd_get_master(parent); uint64_t cur_offset = 0; int i, ret; - printk(KERN_NOTICE "Creating %d MTD partitions on \"%s\":\n", nbparts, master->name); + printk(KERN_NOTICE "Creating %d MTD partitions on \"%s\":\n", + nbparts, parent->name); for (i = 0; i < nbparts; i++) { - slave = allocate_partition(master, parts + i, i, cur_offset); - if (IS_ERR(slave)) { - ret = PTR_ERR(slave); + child = allocate_partition(parent, parts + i, i, cur_offset); + if (IS_ERR(child)) { + ret = PTR_ERR(child); goto err_del_partitions; } - mutex_lock(&mtd_partitions_mutex); - list_add(&slave->list, &mtd_partitions); - mutex_unlock(&mtd_partitions_mutex); + mutex_lock(&master->master.partitions_lock); + list_add_tail(&child->part.node, &parent->partitions); + mutex_unlock(&master->master.partitions_lock); - ret = add_mtd_device(&slave->mtd); + ret = add_mtd_device(child); if (ret) { - mutex_lock(&mtd_partitions_mutex); - list_del(&slave->list); - mutex_unlock(&mtd_partitions_mutex); + mutex_lock(&master->master.partitions_lock); + list_del(&child->part.node); + mutex_unlock(&master->master.partitions_lock); - free_partition(slave); + free_partition(child); goto err_del_partitions; } - mtd_add_partition_attrs(slave); - /* Look for subpartitions */ - parse_mtd_partitions(&slave->mtd, parts[i].types, NULL); + mtd_add_partition_attrs(child); - cur_offset = slave->offset + slave->mtd.size; + /* Look for subpartitions */ + parse_mtd_partitions(child, parts[i].types, NULL); + + cur_offset = child->part.offset + child->size; } return 0; @@ -1023,29 +700,11 @@ void mtd_part_parser_cleanup(struct mtd_partitions *parts) } } -int mtd_is_partition(const struct mtd_info *mtd) -{ - struct mtd_part *part; - int ispart = 0; - - mutex_lock(&mtd_partitions_mutex); - list_for_each_entry(part, &mtd_partitions, list) - if (&part->mtd == mtd) { - ispart = 1; - break; - } - mutex_unlock(&mtd_partitions_mutex); - - return ispart; -} -EXPORT_SYMBOL_GPL(mtd_is_partition); - /* Returns the size of the entire flash chip */ uint64_t mtd_get_device_size(const struct mtd_info *mtd) { - if (!mtd_is_partition(mtd)) - return mtd->size; + struct mtd_info *master = mtd_get_master((struct mtd_info *)mtd); - return mtd_get_device_size(mtd_to_part(mtd)->parent); + return master->size; } EXPORT_SYMBOL_GPL(mtd_get_device_size); diff --git a/drivers/mtd/nand/onenand/onenand_base.c b/drivers/mtd/nand/onenand/onenand_base.c index d5326d19b136..ec18ade33262 100644 --- a/drivers/mtd/nand/onenand/onenand_base.c +++ b/drivers/mtd/nand/onenand/onenand_base.c @@ -3259,7 +3259,7 @@ static void onenand_check_features(struct mtd_info *mtd) switch (density) { case ONENAND_DEVICE_DENSITY_8Gb: this->options |= ONENAND_HAS_NOP_1; - /* fall through */ + fallthrough; case ONENAND_DEVICE_DENSITY_4Gb: if (ONENAND_IS_DDP(this)) this->options |= ONENAND_HAS_2PLANE; diff --git a/drivers/mtd/nand/raw/ams-delta.c b/drivers/mtd/nand/raw/ams-delta.c index 8312182088c1..d66dab25df20 100644 --- a/drivers/mtd/nand/raw/ams-delta.c +++ b/drivers/mtd/nand/raw/ams-delta.c @@ -19,15 +19,17 @@ #include #include #include +#include #include #include +#include #include #include /* * MTD structure for E3 (Delta) */ -struct ams_delta_nand { +struct gpio_nand { struct nand_controller base; struct nand_chip nand_chip; struct gpio_desc *gpiod_rdy; @@ -39,41 +41,20 @@ struct ams_delta_nand { struct gpio_desc *gpiod_cle; struct gpio_descs *data_gpiods; bool data_in; + unsigned int tRP; + unsigned int tWP; + u8 (*io_read)(struct gpio_nand *this); + void (*io_write)(struct gpio_nand *this, u8 byte); }; -/* - * Define partitions for flash devices - */ - -static const struct mtd_partition partition_info[] = { - { .name = "Kernel", - .offset = 0, - .size = 3 * SZ_1M + SZ_512K }, - { .name = "u-boot", - .offset = 3 * SZ_1M + SZ_512K, - .size = SZ_256K }, - { .name = "u-boot params", - .offset = 3 * SZ_1M + SZ_512K + SZ_256K, - .size = SZ_256K }, - { .name = "Amstrad LDR", - .offset = 4 * SZ_1M, - .size = SZ_256K }, - { .name = "File system", - .offset = 4 * SZ_1M + 1 * SZ_256K, - .size = 27 * SZ_1M }, - { .name = "PBL reserved", - .offset = 32 * SZ_1M - 3 * SZ_256K, - .size = 3 * SZ_256K }, -}; - -static void ams_delta_write_commit(struct ams_delta_nand *priv) +static void gpio_nand_write_commit(struct gpio_nand *priv) { - gpiod_set_value(priv->gpiod_nwe, 0); - ndelay(40); gpiod_set_value(priv->gpiod_nwe, 1); + ndelay(priv->tWP); + gpiod_set_value(priv->gpiod_nwe, 0); } -static void ams_delta_io_write(struct ams_delta_nand *priv, u8 byte) +static void gpio_nand_io_write(struct gpio_nand *priv, u8 byte) { struct gpio_descs *data_gpiods = priv->data_gpiods; DECLARE_BITMAP(values, BITS_PER_TYPE(byte)) = { byte, }; @@ -81,10 +62,10 @@ static void ams_delta_io_write(struct ams_delta_nand *priv, u8 byte) gpiod_set_raw_array_value(data_gpiods->ndescs, data_gpiods->desc, data_gpiods->info, values); - ams_delta_write_commit(priv); + gpio_nand_write_commit(priv); } -static void ams_delta_dir_output(struct ams_delta_nand *priv, u8 byte) +static void gpio_nand_dir_output(struct gpio_nand *priv, u8 byte) { struct gpio_descs *data_gpiods = priv->data_gpiods; DECLARE_BITMAP(values, BITS_PER_TYPE(byte)) = { byte, }; @@ -94,30 +75,30 @@ static void ams_delta_dir_output(struct ams_delta_nand *priv, u8 byte) gpiod_direction_output_raw(data_gpiods->desc[i], test_bit(i, values)); - ams_delta_write_commit(priv); + gpio_nand_write_commit(priv); priv->data_in = false; } -static u8 ams_delta_io_read(struct ams_delta_nand *priv) +static u8 gpio_nand_io_read(struct gpio_nand *priv) { u8 res; struct gpio_descs *data_gpiods = priv->data_gpiods; DECLARE_BITMAP(values, BITS_PER_TYPE(res)) = { 0, }; - gpiod_set_value(priv->gpiod_nre, 0); - ndelay(40); + gpiod_set_value(priv->gpiod_nre, 1); + ndelay(priv->tRP); gpiod_get_raw_array_value(data_gpiods->ndescs, data_gpiods->desc, data_gpiods->info, values); - gpiod_set_value(priv->gpiod_nre, 1); + gpiod_set_value(priv->gpiod_nre, 0); res = values[0]; return res; } -static void ams_delta_dir_input(struct ams_delta_nand *priv) +static void gpio_nand_dir_input(struct gpio_nand *priv) { struct gpio_descs *data_gpiods = priv->data_gpiods; int i; @@ -128,68 +109,67 @@ static void ams_delta_dir_input(struct ams_delta_nand *priv) priv->data_in = true; } -static void ams_delta_write_buf(struct ams_delta_nand *priv, const u8 *buf, - int len) +static void gpio_nand_write_buf(struct gpio_nand *priv, const u8 *buf, int len) { int i = 0; if (len > 0 && priv->data_in) - ams_delta_dir_output(priv, buf[i++]); + gpio_nand_dir_output(priv, buf[i++]); while (i < len) - ams_delta_io_write(priv, buf[i++]); + priv->io_write(priv, buf[i++]); } -static void ams_delta_read_buf(struct ams_delta_nand *priv, u8 *buf, int len) +static void gpio_nand_read_buf(struct gpio_nand *priv, u8 *buf, int len) { int i; - if (!priv->data_in) - ams_delta_dir_input(priv); + if (priv->data_gpiods && !priv->data_in) + gpio_nand_dir_input(priv); for (i = 0; i < len; i++) - buf[i] = ams_delta_io_read(priv); + buf[i] = priv->io_read(priv); } -static void ams_delta_ctrl_cs(struct ams_delta_nand *priv, bool assert) +static void gpio_nand_ctrl_cs(struct gpio_nand *priv, bool assert) { - gpiod_set_value(priv->gpiod_nce, assert ? 0 : 1); + gpiod_set_value(priv->gpiod_nce, assert); } -static int ams_delta_exec_op(struct nand_chip *this, +static int gpio_nand_exec_op(struct nand_chip *this, const struct nand_operation *op, bool check_only) { - struct ams_delta_nand *priv = nand_get_controller_data(this); + struct gpio_nand *priv = nand_get_controller_data(this); const struct nand_op_instr *instr; int ret = 0; if (check_only) return 0; - ams_delta_ctrl_cs(priv, 1); + gpio_nand_ctrl_cs(priv, 1); for (instr = op->instrs; instr < op->instrs + op->ninstrs; instr++) { switch (instr->type) { case NAND_OP_CMD_INSTR: gpiod_set_value(priv->gpiod_cle, 1); - ams_delta_write_buf(priv, &instr->ctx.cmd.opcode, 1); + gpio_nand_write_buf(priv, &instr->ctx.cmd.opcode, 1); gpiod_set_value(priv->gpiod_cle, 0); break; case NAND_OP_ADDR_INSTR: gpiod_set_value(priv->gpiod_ale, 1); - ams_delta_write_buf(priv, instr->ctx.addr.addrs, + gpio_nand_write_buf(priv, instr->ctx.addr.addrs, instr->ctx.addr.naddrs); gpiod_set_value(priv->gpiod_ale, 0); break; case NAND_OP_DATA_IN_INSTR: - ams_delta_read_buf(priv, instr->ctx.data.buf.in, + gpio_nand_read_buf(priv, instr->ctx.data.buf.in, instr->ctx.data.len); break; case NAND_OP_DATA_OUT_INSTR: - ams_delta_write_buf(priv, instr->ctx.data.buf.out, + gpio_nand_write_buf(priv, instr->ctx.data.buf.out, instr->ctx.data.len); break; @@ -206,28 +186,61 @@ static int ams_delta_exec_op(struct nand_chip *this, break; } - ams_delta_ctrl_cs(priv, 0); + gpio_nand_ctrl_cs(priv, 0); return ret; } -static const struct nand_controller_ops ams_delta_ops = { - .exec_op = ams_delta_exec_op, +static int gpio_nand_setup_data_interface(struct nand_chip *this, int csline, + const struct nand_data_interface *cf) +{ + struct gpio_nand *priv = nand_get_controller_data(this); + const struct nand_sdr_timings *sdr = nand_get_sdr_timings(cf); + struct device *dev = &nand_to_mtd(this)->dev; + + if (IS_ERR(sdr)) + return PTR_ERR(sdr); + + if (csline == NAND_DATA_IFACE_CHECK_ONLY) + return 0; + + if (priv->gpiod_nre) { + priv->tRP = DIV_ROUND_UP(sdr->tRP_min, 1000); + dev_dbg(dev, "using %u ns read pulse width\n", priv->tRP); + } + + priv->tWP = DIV_ROUND_UP(sdr->tWP_min, 1000); + dev_dbg(dev, "using %u ns write pulse width\n", priv->tWP); + + return 0; +} + +static const struct nand_controller_ops gpio_nand_ops = { + .exec_op = gpio_nand_exec_op, + .setup_data_interface = gpio_nand_setup_data_interface, }; /* * Main initialization routine */ -static int ams_delta_init(struct platform_device *pdev) +static int gpio_nand_probe(struct platform_device *pdev) { - struct ams_delta_nand *priv; + struct gpio_nand_platdata *pdata = dev_get_platdata(&pdev->dev); + const struct mtd_partition *partitions = NULL; + int num_partitions = 0; + struct gpio_nand *priv; struct nand_chip *this; struct mtd_info *mtd; - struct gpio_descs *data_gpiods; + int (*probe)(struct platform_device *pdev, struct gpio_nand *priv); int err = 0; + if (pdata) { + partitions = pdata->parts; + num_partitions = pdata->num_parts; + } + /* Allocate memory for MTD device structure and private data */ - priv = devm_kzalloc(&pdev->dev, sizeof(struct ams_delta_nand), + priv = devm_kzalloc(&pdev->dev, sizeof(struct gpio_nand), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -238,6 +251,7 @@ static int ams_delta_init(struct platform_device *pdev) mtd->dev.parent = &pdev->dev; nand_set_controller_data(this, priv); + nand_set_flash_node(this, pdev->dev.of_node); priv->gpiod_rdy = devm_gpiod_get_optional(&pdev->dev, "rdy", GPIOD_IN); if (IS_ERR(priv->gpiod_rdy)) { @@ -251,29 +265,33 @@ static int ams_delta_init(struct platform_device *pdev) platform_set_drvdata(pdev, priv); - /* Set chip enabled, but */ - priv->gpiod_nwp = devm_gpiod_get(&pdev->dev, "nwp", GPIOD_OUT_HIGH); + /* Set chip enabled but write protected */ + priv->gpiod_nwp = devm_gpiod_get_optional(&pdev->dev, "nwp", + GPIOD_OUT_HIGH); if (IS_ERR(priv->gpiod_nwp)) { err = PTR_ERR(priv->gpiod_nwp); dev_err(&pdev->dev, "NWP GPIO request failed (%d)\n", err); return err; } - priv->gpiod_nce = devm_gpiod_get(&pdev->dev, "nce", GPIOD_OUT_HIGH); + priv->gpiod_nce = devm_gpiod_get_optional(&pdev->dev, "nce", + GPIOD_OUT_LOW); if (IS_ERR(priv->gpiod_nce)) { err = PTR_ERR(priv->gpiod_nce); dev_err(&pdev->dev, "NCE GPIO request failed (%d)\n", err); return err; } - priv->gpiod_nre = devm_gpiod_get(&pdev->dev, "nre", GPIOD_OUT_HIGH); + priv->gpiod_nre = devm_gpiod_get_optional(&pdev->dev, "nre", + GPIOD_OUT_LOW); if (IS_ERR(priv->gpiod_nre)) { err = PTR_ERR(priv->gpiod_nre); dev_err(&pdev->dev, "NRE GPIO request failed (%d)\n", err); return err; } - priv->gpiod_nwe = devm_gpiod_get(&pdev->dev, "nwe", GPIOD_OUT_HIGH); + priv->gpiod_nwe = devm_gpiod_get_optional(&pdev->dev, "nwe", + GPIOD_OUT_LOW); if (IS_ERR(priv->gpiod_nwe)) { err = PTR_ERR(priv->gpiod_nwe); dev_err(&pdev->dev, "NWE GPIO request failed (%d)\n", err); @@ -295,28 +313,62 @@ static int ams_delta_init(struct platform_device *pdev) } /* Request array of data pins, initialize them as input */ - data_gpiods = devm_gpiod_get_array(&pdev->dev, "data", GPIOD_IN); - if (IS_ERR(data_gpiods)) { - err = PTR_ERR(data_gpiods); + priv->data_gpiods = devm_gpiod_get_array_optional(&pdev->dev, "data", + GPIOD_IN); + if (IS_ERR(priv->data_gpiods)) { + err = PTR_ERR(priv->data_gpiods); dev_err(&pdev->dev, "data GPIO request failed: %d\n", err); return err; } - priv->data_gpiods = data_gpiods; - priv->data_in = true; + if (priv->data_gpiods) { + if (!priv->gpiod_nwe) { + dev_err(&pdev->dev, + "mandatory NWE pin not provided by platform\n"); + return -ENODEV; + } - /* Initialize the NAND controller object embedded in ams_delta_nand. */ - priv->base.ops = &ams_delta_ops; + priv->io_read = gpio_nand_io_read; + priv->io_write = gpio_nand_io_write; + priv->data_in = true; + } + + if (pdev->id_entry) + probe = (void *) pdev->id_entry->driver_data; + else + probe = of_device_get_match_data(&pdev->dev); + if (probe) + err = probe(pdev, priv); + if (err) + return err; + + if (!priv->io_read || !priv->io_write) { + dev_err(&pdev->dev, "incomplete device configuration\n"); + return -ENODEV; + } + + /* Initialize the NAND controller object embedded in gpio_nand. */ + priv->base.ops = &gpio_nand_ops; nand_controller_init(&priv->base); this->controller = &priv->base; + /* + * FIXME: We should release write protection only after nand_scan() to + * be on the safe side but we can't do that until we have a generic way + * to assert/deassert WP from the core. Even if the core shouldn't + * write things in the nand_scan() path, it should have control on this + * pin just in case we ever need to disable write protection during + * chip detection/initialization. + */ + /* Release write protection */ + gpiod_set_value(priv->gpiod_nwp, 0); + /* Scan to find existence of the device */ err = nand_scan(this, 1); if (err) return err; /* Register the partitions */ - err = mtd_device_register(mtd, partition_info, - ARRAY_SIZE(partition_info)); + err = mtd_device_register(mtd, partitions, num_partitions); if (err) goto err_nand_cleanup; @@ -331,26 +383,47 @@ static int ams_delta_init(struct platform_device *pdev) /* * Clean up routine */ -static int ams_delta_cleanup(struct platform_device *pdev) +static int gpio_nand_remove(struct platform_device *pdev) { - struct ams_delta_nand *priv = platform_get_drvdata(pdev); + struct gpio_nand *priv = platform_get_drvdata(pdev); struct mtd_info *mtd = nand_to_mtd(&priv->nand_chip); + /* Apply write protection */ + gpiod_set_value(priv->gpiod_nwp, 1); + /* Unregister device */ nand_release(mtd_to_nand(mtd)); return 0; } -static struct platform_driver ams_delta_nand_driver = { - .probe = ams_delta_init, - .remove = ams_delta_cleanup, +static const struct of_device_id gpio_nand_of_id_table[] = { + { + /* sentinel */ + }, +}; +MODULE_DEVICE_TABLE(of, gpio_nand_of_id_table); + +static const struct platform_device_id gpio_nand_plat_id_table[] = { + { + .name = "ams-delta-nand", + }, { + /* sentinel */ + }, +}; +MODULE_DEVICE_TABLE(platform, gpio_nand_plat_id_table); + +static struct platform_driver gpio_nand_driver = { + .probe = gpio_nand_probe, + .remove = gpio_nand_remove, + .id_table = gpio_nand_plat_id_table, .driver = { .name = "ams-delta-nand", + .of_match_table = of_match_ptr(gpio_nand_of_id_table), }, }; -module_platform_driver(ams_delta_nand_driver); +module_platform_driver(gpio_nand_driver); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Jonathan McDowell "); diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c index 44518dada75b..e4e3ceeac38f 100644 --- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c +++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c @@ -102,6 +102,45 @@ struct brcm_nand_dma_desc { #define NAND_CTRL_RDY (INTFC_CTLR_READY | INTFC_FLASH_READY) #define NAND_POLL_STATUS_TIMEOUT_MS 100 +#define EDU_CMD_WRITE 0x00 +#define EDU_CMD_READ 0x01 +#define EDU_STATUS_ACTIVE BIT(0) +#define EDU_ERR_STATUS_ERRACK BIT(0) +#define EDU_DONE_MASK GENMASK(1, 0) + +#define EDU_CONFIG_MODE_NAND BIT(0) +#define EDU_CONFIG_SWAP_BYTE BIT(1) +#ifdef CONFIG_CPU_BIG_ENDIAN +#define EDU_CONFIG_SWAP_CFG EDU_CONFIG_SWAP_BYTE +#else +#define EDU_CONFIG_SWAP_CFG 0 +#endif + +/* edu registers */ +enum edu_reg { + EDU_CONFIG = 0, + EDU_DRAM_ADDR, + EDU_EXT_ADDR, + EDU_LENGTH, + EDU_CMD, + EDU_STOP, + EDU_STATUS, + EDU_DONE, + EDU_ERR_STATUS, +}; + +static const u16 edu_regs[] = { + [EDU_CONFIG] = 0x00, + [EDU_DRAM_ADDR] = 0x04, + [EDU_EXT_ADDR] = 0x08, + [EDU_LENGTH] = 0x0c, + [EDU_CMD] = 0x10, + [EDU_STOP] = 0x14, + [EDU_STATUS] = 0x18, + [EDU_DONE] = 0x1c, + [EDU_ERR_STATUS] = 0x20, +}; + /* flash_dma registers */ enum flash_dma_reg { FLASH_DMA_REVISION = 0, @@ -167,6 +206,8 @@ enum { BRCMNAND_HAS_WP = BIT(3), }; +struct brcmnand_host; + struct brcmnand_controller { struct device *dev; struct nand_controller controller; @@ -185,17 +226,32 @@ struct brcmnand_controller { int cmd_pending; bool dma_pending; + bool edu_pending; struct completion done; struct completion dma_done; + struct completion edu_done; /* List of NAND hosts (one for each chip-select) */ struct list_head host_list; + /* EDU info, per-transaction */ + const u16 *edu_offsets; + void __iomem *edu_base; + int edu_irq; + int edu_count; + u64 edu_dram_addr; + u32 edu_ext_addr; + u32 edu_cmd; + u32 edu_config; + /* flash_dma reg */ const u16 *flash_dma_offsets; struct brcm_nand_dma_desc *dma_desc; dma_addr_t dma_pa; + int (*dma_trans)(struct brcmnand_host *host, u64 addr, u32 *buf, + u32 len, u8 dma_cmd); + /* in-memory cache of the FLASH_CACHE, used only for some commands */ u8 flash_cache[FC_BYTES]; @@ -216,6 +272,7 @@ struct brcmnand_controller { u32 nand_cs_nand_xor; u32 corr_stat_threshold; u32 flash_dma_mode; + u32 flash_edu_mode; bool pio_poll_mode; }; @@ -657,6 +714,22 @@ static inline void brcmnand_write_fc(struct brcmnand_controller *ctrl, __raw_writel(val, ctrl->nand_fc + word * 4); } +static inline void edu_writel(struct brcmnand_controller *ctrl, + enum edu_reg reg, u32 val) +{ + u16 offs = ctrl->edu_offsets[reg]; + + brcmnand_writel(val, ctrl->edu_base + offs); +} + +static inline u32 edu_readl(struct brcmnand_controller *ctrl, + enum edu_reg reg) +{ + u16 offs = ctrl->edu_offsets[reg]; + + return brcmnand_readl(ctrl->edu_base + offs); +} + static void brcmnand_clear_ecc_addr(struct brcmnand_controller *ctrl) { @@ -926,6 +999,16 @@ static inline bool has_flash_dma(struct brcmnand_controller *ctrl) return ctrl->flash_dma_base; } +static inline bool has_edu(struct brcmnand_controller *ctrl) +{ + return ctrl->edu_base; +} + +static inline bool use_dma(struct brcmnand_controller *ctrl) +{ + return has_flash_dma(ctrl) || has_edu(ctrl); +} + static inline void disable_ctrl_irqs(struct brcmnand_controller *ctrl) { if (ctrl->pio_poll_mode) @@ -1299,6 +1382,52 @@ static int write_oob_to_regs(struct brcmnand_controller *ctrl, int i, return tbytes; } +static void brcmnand_edu_init(struct brcmnand_controller *ctrl) +{ + /* initialize edu */ + edu_writel(ctrl, EDU_ERR_STATUS, 0); + edu_readl(ctrl, EDU_ERR_STATUS); + edu_writel(ctrl, EDU_DONE, 0); + edu_writel(ctrl, EDU_DONE, 0); + edu_writel(ctrl, EDU_DONE, 0); + edu_writel(ctrl, EDU_DONE, 0); + edu_readl(ctrl, EDU_DONE); +} + +/* edu irq */ +static irqreturn_t brcmnand_edu_irq(int irq, void *data) +{ + struct brcmnand_controller *ctrl = data; + + if (ctrl->edu_count) { + ctrl->edu_count--; + while (!(edu_readl(ctrl, EDU_DONE) & EDU_DONE_MASK)) + udelay(1); + edu_writel(ctrl, EDU_DONE, 0); + edu_readl(ctrl, EDU_DONE); + } + + if (ctrl->edu_count) { + ctrl->edu_dram_addr += FC_BYTES; + ctrl->edu_ext_addr += FC_BYTES; + + edu_writel(ctrl, EDU_DRAM_ADDR, (u32)ctrl->edu_dram_addr); + edu_readl(ctrl, EDU_DRAM_ADDR); + edu_writel(ctrl, EDU_EXT_ADDR, ctrl->edu_ext_addr); + edu_readl(ctrl, EDU_EXT_ADDR); + + mb(); /* flush previous writes */ + edu_writel(ctrl, EDU_CMD, ctrl->edu_cmd); + edu_readl(ctrl, EDU_CMD); + + return IRQ_HANDLED; + } + + complete(&ctrl->edu_done); + + return IRQ_HANDLED; +} + static irqreturn_t brcmnand_ctlrdy_irq(int irq, void *data) { struct brcmnand_controller *ctrl = data; @@ -1307,6 +1436,16 @@ static irqreturn_t brcmnand_ctlrdy_irq(int irq, void *data) if (ctrl->dma_pending) return IRQ_HANDLED; + /* check if you need to piggy back on the ctrlrdy irq */ + if (ctrl->edu_pending) { + if (irq == ctrl->irq && ((int)ctrl->edu_irq >= 0)) + /* Discard interrupts while using dedicated edu irq */ + return IRQ_HANDLED; + + /* no registered edu irq, call handler */ + return brcmnand_edu_irq(irq, data); + } + complete(&ctrl->done); return IRQ_HANDLED; } @@ -1644,6 +1783,81 @@ static void brcmnand_write_buf(struct nand_chip *chip, const uint8_t *buf, } } +/** + * Kick EDU engine + */ +static int brcmnand_edu_trans(struct brcmnand_host *host, u64 addr, u32 *buf, + u32 len, u8 cmd) +{ + struct brcmnand_controller *ctrl = host->ctrl; + unsigned long timeo = msecs_to_jiffies(200); + int ret = 0; + int dir = (cmd == CMD_PAGE_READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE); + u8 edu_cmd = (cmd == CMD_PAGE_READ ? EDU_CMD_READ : EDU_CMD_WRITE); + unsigned int trans = len >> FC_SHIFT; + dma_addr_t pa; + + pa = dma_map_single(ctrl->dev, buf, len, dir); + if (dma_mapping_error(ctrl->dev, pa)) { + dev_err(ctrl->dev, "unable to map buffer for EDU DMA\n"); + return -ENOMEM; + } + + ctrl->edu_pending = true; + ctrl->edu_dram_addr = pa; + ctrl->edu_ext_addr = addr; + ctrl->edu_cmd = edu_cmd; + ctrl->edu_count = trans; + + edu_writel(ctrl, EDU_DRAM_ADDR, (u32)ctrl->edu_dram_addr); + edu_readl(ctrl, EDU_DRAM_ADDR); + edu_writel(ctrl, EDU_EXT_ADDR, ctrl->edu_ext_addr); + edu_readl(ctrl, EDU_EXT_ADDR); + edu_writel(ctrl, EDU_LENGTH, FC_BYTES); + edu_readl(ctrl, EDU_LENGTH); + + /* Start edu engine */ + mb(); /* flush previous writes */ + edu_writel(ctrl, EDU_CMD, ctrl->edu_cmd); + edu_readl(ctrl, EDU_CMD); + + if (wait_for_completion_timeout(&ctrl->edu_done, timeo) <= 0) { + dev_err(ctrl->dev, + "timeout waiting for EDU; status %#x, error status %#x\n", + edu_readl(ctrl, EDU_STATUS), + edu_readl(ctrl, EDU_ERR_STATUS)); + } + + dma_unmap_single(ctrl->dev, pa, len, dir); + + /* for program page check NAND status */ + if (((brcmnand_read_reg(ctrl, BRCMNAND_INTFC_STATUS) & + INTFC_FLASH_STATUS) & NAND_STATUS_FAIL) && + edu_cmd == EDU_CMD_WRITE) { + dev_info(ctrl->dev, "program failed at %llx\n", + (unsigned long long)addr); + ret = -EIO; + } + + /* Make sure the EDU status is clean */ + if (edu_readl(ctrl, EDU_STATUS) & EDU_STATUS_ACTIVE) + dev_warn(ctrl->dev, "EDU still active: %#x\n", + edu_readl(ctrl, EDU_STATUS)); + + if (unlikely(edu_readl(ctrl, EDU_ERR_STATUS) & EDU_ERR_STATUS_ERRACK)) { + dev_warn(ctrl->dev, "EDU RBUS error at addr %llx\n", + (unsigned long long)addr); + ret = -EIO; + } + + ctrl->edu_pending = false; + brcmnand_edu_init(ctrl); + edu_writel(ctrl, EDU_STOP, 0); /* force stop */ + edu_readl(ctrl, EDU_STOP); + + return ret; +} + /** * Construct a FLASH_DMA descriptor as part of a linked list. You must know the * following ahead of time: @@ -1850,9 +2064,11 @@ static int brcmnand_read(struct mtd_info *mtd, struct nand_chip *chip, try_dmaread: brcmnand_clear_ecc_addr(ctrl); - if (has_flash_dma(ctrl) && !oob && flash_dma_buf_ok(buf)) { - err = brcmnand_dma_trans(host, addr, buf, trans * FC_BYTES, - CMD_PAGE_READ); + if (ctrl->dma_trans && !oob && flash_dma_buf_ok(buf)) { + err = ctrl->dma_trans(host, addr, buf, + trans * FC_BYTES, + CMD_PAGE_READ); + if (err) { if (mtd_is_bitflip_or_eccerr(err)) err_addr = addr; @@ -1988,10 +2204,12 @@ static int brcmnand_write(struct mtd_info *mtd, struct nand_chip *chip, for (i = 0; i < ctrl->max_oob; i += 4) oob_reg_write(ctrl, i, 0xffffffff); - if (has_flash_dma(ctrl) && !oob && flash_dma_buf_ok(buf)) { - if (brcmnand_dma_trans(host, addr, (u32 *)buf, - mtd->writesize, CMD_PROGRAM_PAGE)) + if (use_dma(ctrl) && !oob && flash_dma_buf_ok(buf)) { + if (ctrl->dma_trans(host, addr, (u32 *)buf, mtd->writesize, + CMD_PROGRAM_PAGE)) + ret = -EIO; + goto out; } @@ -2494,6 +2712,8 @@ static int brcmnand_suspend(struct device *dev) if (has_flash_dma(ctrl)) ctrl->flash_dma_mode = flash_dma_readl(ctrl, FLASH_DMA_MODE); + else if (has_edu(ctrl)) + ctrl->edu_config = edu_readl(ctrl, EDU_CONFIG); return 0; } @@ -2508,6 +2728,14 @@ static int brcmnand_resume(struct device *dev) flash_dma_writel(ctrl, FLASH_DMA_ERROR_STATUS, 0); } + if (has_edu(ctrl)) + ctrl->edu_config = edu_readl(ctrl, EDU_CONFIG); + else { + edu_writel(ctrl, EDU_CONFIG, ctrl->edu_config); + edu_readl(ctrl, EDU_CONFIG); + brcmnand_edu_init(ctrl); + } + brcmnand_write_reg(ctrl, BRCMNAND_CS_SELECT, ctrl->nand_cs_nand_select); brcmnand_write_reg(ctrl, BRCMNAND_CS_XOR, ctrl->nand_cs_nand_xor); brcmnand_write_reg(ctrl, BRCMNAND_CORR_THRESHOLD, @@ -2553,6 +2781,49 @@ MODULE_DEVICE_TABLE(of, brcmnand_of_match); /*********************************************************************** * Platform driver setup (per controller) ***********************************************************************/ +static int brcmnand_edu_setup(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct brcmnand_controller *ctrl = dev_get_drvdata(&pdev->dev); + struct resource *res; + int ret; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "flash-edu"); + if (res) { + ctrl->edu_base = devm_ioremap_resource(dev, res); + if (IS_ERR(ctrl->edu_base)) + return PTR_ERR(ctrl->edu_base); + + ctrl->edu_offsets = edu_regs; + + edu_writel(ctrl, EDU_CONFIG, EDU_CONFIG_MODE_NAND | + EDU_CONFIG_SWAP_CFG); + edu_readl(ctrl, EDU_CONFIG); + + /* initialize edu */ + brcmnand_edu_init(ctrl); + + ctrl->edu_irq = platform_get_irq_optional(pdev, 1); + if (ctrl->edu_irq < 0) { + dev_warn(dev, + "FLASH EDU enabled, using ctlrdy irq\n"); + } else { + ret = devm_request_irq(dev, ctrl->edu_irq, + brcmnand_edu_irq, 0, + "brcmnand-edu", ctrl); + if (ret < 0) { + dev_err(ctrl->dev, "can't allocate IRQ %d: error %d\n", + ctrl->edu_irq, ret); + return ret; + } + + dev_info(dev, "FLASH EDU enabled using irq %u\n", + ctrl->edu_irq); + } + } + + return 0; +} int brcmnand_probe(struct platform_device *pdev, struct brcmnand_soc *soc) { @@ -2578,6 +2849,7 @@ int brcmnand_probe(struct platform_device *pdev, struct brcmnand_soc *soc) init_completion(&ctrl->done); init_completion(&ctrl->dma_done); + init_completion(&ctrl->edu_done); nand_controller_init(&ctrl->controller); ctrl->controller.ops = &brcmnand_controller_ops; INIT_LIST_HEAD(&ctrl->host_list); @@ -2675,6 +2947,15 @@ int brcmnand_probe(struct platform_device *pdev, struct brcmnand_soc *soc) } dev_info(dev, "enabling FLASH_DMA\n"); + /* set flash dma transfer function to call */ + ctrl->dma_trans = brcmnand_dma_trans; + } else { + ret = brcmnand_edu_setup(pdev); + if (ret < 0) + goto err; + + /* set edu transfer function to call */ + ctrl->dma_trans = brcmnand_edu_trans; } /* Disable automatic device ID config, direct addressing */ diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c index f6c7102a1e32..efddc5c68afb 100644 --- a/drivers/mtd/nand/raw/cadence-nand-controller.c +++ b/drivers/mtd/nand/raw/cadence-nand-controller.c @@ -30,7 +30,6 @@ * Generic mode is used for executing rest of commands. */ -#define MAX_OOB_SIZE_PER_SECTOR 32 #define MAX_ADDRESS_CYC 6 #define MAX_ERASE_ADDRESS_CYC 3 #define MAX_DATA_SIZE 0xFFFC @@ -190,6 +189,7 @@ /* BCH Engine identification register 3. */ #define BCH_CFG_3 0x844 +#define BCH_CFG_3_METADATA_SIZE GENMASK(23, 16) /* Ready/Busy# line status. */ #define RBN_SETINGS 0x1004 @@ -499,6 +499,7 @@ struct cdns_nand_ctrl { unsigned long assigned_cs; struct list_head chips; + u8 bch_metadata_size; }; struct cdns_nand_chip { @@ -997,6 +998,7 @@ static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl, return status; cadence_nand_reset_irq(cdns_ctrl); + reinit_completion(&cdns_ctrl->complete); writel_relaxed((u32)cdns_ctrl->dma_cdma_desc, cdns_ctrl->reg + CMD_REG2); @@ -1077,6 +1079,14 @@ static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl) int max_step_size = 0, nstrengths, i; u32 reg; + reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_3); + cdns_ctrl->bch_metadata_size = FIELD_GET(BCH_CFG_3_METADATA_SIZE, reg); + if (cdns_ctrl->bch_metadata_size < 4) { + dev_err(cdns_ctrl->dev, + "Driver needs at least 4 bytes of BCH meta data\n"); + return -EIO; + } + reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_0); cdns_ctrl->ecc_strengths[0] = FIELD_GET(BCH_CFG_0_CORR_CAP_0, reg); cdns_ctrl->ecc_strengths[1] = FIELD_GET(BCH_CFG_0_CORR_CAP_1, reg); @@ -1170,7 +1180,8 @@ static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl) writel_relaxed(0xFFFFFFFF, cdns_ctrl->reg + INTR_STATUS); cadence_nand_get_caps(cdns_ctrl); - cadence_nand_read_bch_caps(cdns_ctrl); + if (cadence_nand_read_bch_caps(cdns_ctrl)) + return -EIO; /* * Set IO width access to 8. @@ -2585,9 +2596,8 @@ int cadence_nand_attach_chip(struct nand_chip *chip) { struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller); struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip); - u32 ecc_size = cdns_chip->sector_count * chip->ecc.bytes; + u32 ecc_size; struct mtd_info *mtd = nand_to_mtd(chip); - u32 max_oob_data_size; int ret; if (chip->options & NAND_BUSWIDTH_16) { @@ -2603,12 +2613,9 @@ int cadence_nand_attach_chip(struct nand_chip *chip) chip->options |= NAND_NO_SUBPAGE_WRITE; cdns_chip->bbm_offs = chip->badblockpos; - if (chip->options & NAND_BUSWIDTH_16) { - cdns_chip->bbm_offs &= ~0x01; - cdns_chip->bbm_len = 2; - } else { - cdns_chip->bbm_len = 1; - } + cdns_chip->bbm_offs &= ~0x01; + /* this value should be even number */ + cdns_chip->bbm_len = 2; ret = nand_ecc_choose_conf(chip, &cdns_ctrl->ecc_caps, @@ -2625,13 +2632,12 @@ int cadence_nand_attach_chip(struct nand_chip *chip) /* Error correction configuration. */ cdns_chip->sector_size = chip->ecc.size; cdns_chip->sector_count = mtd->writesize / cdns_chip->sector_size; + ecc_size = cdns_chip->sector_count * chip->ecc.bytes; cdns_chip->avail_oob_size = mtd->oobsize - ecc_size; - max_oob_data_size = MAX_OOB_SIZE_PER_SECTOR; - - if (cdns_chip->avail_oob_size > max_oob_data_size) - cdns_chip->avail_oob_size = max_oob_data_size; + if (cdns_chip->avail_oob_size > cdns_ctrl->bch_metadata_size) + cdns_chip->avail_oob_size = cdns_ctrl->bch_metadata_size; if ((cdns_chip->avail_oob_size + cdns_chip->bbm_len + ecc_size) > mtd->oobsize) diff --git a/drivers/mtd/nand/raw/denali.c b/drivers/mtd/nand/raw/denali.c index fafd0a0aa8e2..6a6c919b2569 100644 --- a/drivers/mtd/nand/raw/denali.c +++ b/drivers/mtd/nand/raw/denali.c @@ -1317,6 +1317,7 @@ int denali_init(struct denali_controller *denali) iowrite32(CHIP_EN_DONT_CARE__FLAG, denali->reg + CHIP_ENABLE_DONT_CARE); iowrite32(ECC_ENABLE__FLAG, denali->reg + ECC_ENABLE); iowrite32(0xffff, denali->reg + SPARE_AREA_MARKER); + iowrite32(WRITE_PROTECT__FLAG, denali->reg + WRITE_PROTECT); denali_clear_irq_all(denali); diff --git a/drivers/mtd/nand/raw/denali.h b/drivers/mtd/nand/raw/denali.h index e5cdcda56d14..ac46eb7956ce 100644 --- a/drivers/mtd/nand/raw/denali.h +++ b/drivers/mtd/nand/raw/denali.h @@ -328,7 +328,7 @@ struct denali_chip { struct nand_chip chip; struct list_head node; unsigned int nsels; - struct denali_chip_sel sels[0]; + struct denali_chip_sel sels[]; }; /** diff --git a/drivers/mtd/nand/raw/diskonchip.c b/drivers/mtd/nand/raw/diskonchip.c index c0e1a8ebe820..c2a391ad2c35 100644 --- a/drivers/mtd/nand/raw/diskonchip.c +++ b/drivers/mtd/nand/raw/diskonchip.c @@ -1169,7 +1169,7 @@ static inline int __init inftl_partscan(struct mtd_info *mtd, struct mtd_partiti " NoOfBootImageBlocks = %d\n" " NoOfBinaryPartitions = %d\n" " NoOfBDTLPartitions = %d\n" - " BlockMultiplerBits = %d\n" + " BlockMultiplierBits = %d\n" " FormatFlgs = %d\n" " OsakVersion = %d.%d.%d.%d\n" " PercentUsed = %d\n", @@ -1482,7 +1482,7 @@ static int __init doc_probe(unsigned long physadr) break; case DOC_ChipID_DocMilPlus32: pr_err("DiskOnChip Millennium Plus 32MB is not supported, ignoring.\n"); - /* fall through */ + fallthrough; default: ret = -ENODEV; goto notfound; diff --git a/drivers/mtd/nand/raw/fsl_elbc_nand.c b/drivers/mtd/nand/raw/fsl_elbc_nand.c index 634c550db13a..e1dc675b12bb 100644 --- a/drivers/mtd/nand/raw/fsl_elbc_nand.c +++ b/drivers/mtd/nand/raw/fsl_elbc_nand.c @@ -324,8 +324,7 @@ static void fsl_elbc_cmdfunc(struct nand_chip *chip, unsigned int command, /* READ0 and READ1 read the entire buffer to use hardware ECC. */ case NAND_CMD_READ1: column += 256; - - /* fall-through */ + fallthrough; case NAND_CMD_READ0: dev_dbg(priv->dev, "fsl_elbc_cmdfunc: NAND_CMD_READ0, page_addr:" diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c index b9d5d55a5edb..53b00c841aec 100644 --- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c +++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c @@ -1148,20 +1148,21 @@ static int acquire_dma_channels(struct gpmi_nand_data *this) { struct platform_device *pdev = this->pdev; struct dma_chan *dma_chan; + int ret = 0; /* request dma channel */ - dma_chan = dma_request_slave_channel(&pdev->dev, "rx-tx"); - if (!dma_chan) { - dev_err(this->dev, "Failed to request DMA channel.\n"); - goto acquire_err; + dma_chan = dma_request_chan(&pdev->dev, "rx-tx"); + if (IS_ERR(dma_chan)) { + ret = PTR_ERR(dma_chan); + if (ret != -EPROBE_DEFER) + dev_err(this->dev, "DMA channel request failed: %d\n", + ret); + release_dma_channels(this); + } else { + this->dma_chans[0] = dma_chan; } - this->dma_chans[0] = dma_chan; - return 0; - -acquire_err: - release_dma_channels(this); - return -EINVAL; + return ret; } static int gpmi_get_clks(struct gpmi_nand_data *this) diff --git a/drivers/mtd/nand/raw/ingenic/Kconfig b/drivers/mtd/nand/raw/ingenic/Kconfig index e30feb56b650..96c5ae8b1bbc 100644 --- a/drivers/mtd/nand/raw/ingenic/Kconfig +++ b/drivers/mtd/nand/raw/ingenic/Kconfig @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config MTD_NAND_JZ4780 tristate "JZ4780 NAND controller" + depends on MIPS || COMPILE_TEST depends on JZ4780_NEMC help Enables support for NAND Flash connected to the NEMC on JZ4780 SoC diff --git a/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c b/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c index c954189606f6..8e22cd6ec71f 100644 --- a/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c +++ b/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c @@ -124,7 +124,6 @@ int ingenic_ecc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct ingenic_ecc *ecc; - struct resource *res; ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL); if (!ecc) @@ -134,8 +133,7 @@ int ingenic_ecc_probe(struct platform_device *pdev) if (!ecc->ops) return -EINVAL; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - ecc->base = devm_ioremap_resource(dev, res); + ecc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ecc->base)) return PTR_ERR(ecc->base); diff --git a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c index 49afebee50db..935c4902ada7 100644 --- a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c +++ b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c @@ -253,7 +253,7 @@ static int ingenic_nand_attach_chip(struct nand_chip *chip) chip->ecc.hwctl = ingenic_nand_ecc_hwctl; chip->ecc.calculate = ingenic_nand_ecc_calculate; chip->ecc.correct = ingenic_nand_ecc_correct; - /* fall through */ + fallthrough; case NAND_ECC_SOFT: dev_info(nfc->dev, "using %s (strength %d, size %d, bytes %d)\n", (nfc->ecc) ? "hardware ECC" : "software ECC", diff --git a/drivers/mtd/nand/raw/ingenic/jz4725b_bch.c b/drivers/mtd/nand/raw/ingenic/jz4725b_bch.c index 6c852eae09cf..2d0e0a2192ae 100644 --- a/drivers/mtd/nand/raw/ingenic/jz4725b_bch.c +++ b/drivers/mtd/nand/raw/ingenic/jz4725b_bch.c @@ -145,10 +145,10 @@ static void jz4725b_bch_read_parity(struct ingenic_ecc *bch, u8 *buf, switch (size8) { case 3: dest8[2] = (val >> 16) & 0xff; - /* fall-through */ + fallthrough; case 2: dest8[1] = (val >> 8) & 0xff; - /* fall-through */ + fallthrough; case 1: dest8[0] = val & 0xff; break; diff --git a/drivers/mtd/nand/raw/ingenic/jz4780_bch.c b/drivers/mtd/nand/raw/ingenic/jz4780_bch.c index 079266a0d6cf..d67dbfff76cc 100644 --- a/drivers/mtd/nand/raw/ingenic/jz4780_bch.c +++ b/drivers/mtd/nand/raw/ingenic/jz4780_bch.c @@ -123,10 +123,10 @@ static void jz4780_bch_read_parity(struct ingenic_ecc *bch, void *buf, switch (size8) { case 3: dest8[2] = (val >> 16) & 0xff; - /* fall through */ + fallthrough; case 2: dest8[1] = (val >> 8) & 0xff; - /* fall through */ + fallthrough; case 1: dest8[0] = val & 0xff; break; diff --git a/drivers/mtd/nand/raw/internals.h b/drivers/mtd/nand/raw/internals.h index cba6fe7dd8c4..9d0caadf940e 100644 --- a/drivers/mtd/nand/raw/internals.h +++ b/drivers/mtd/nand/raw/internals.h @@ -30,6 +30,7 @@ #define NAND_MFR_SAMSUNG 0xec #define NAND_MFR_SANDISK 0x45 #define NAND_MFR_STMICRO 0x20 +/* Kioxia is new name of Toshiba memory. */ #define NAND_MFR_TOSHIBA 0x98 #define NAND_MFR_WINBOND 0xef diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c index fb5abdcfb007..179f0ca585f8 100644 --- a/drivers/mtd/nand/raw/marvell_nand.c +++ b/drivers/mtd/nand/raw/marvell_nand.c @@ -334,7 +334,7 @@ struct marvell_nand_chip { int addr_cyc; int selected_die; unsigned int nsels; - struct marvell_nand_chip_sel sels[0]; + struct marvell_nand_chip_sel sels[]; }; static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip) @@ -2743,16 +2743,21 @@ static int marvell_nfc_init_dma(struct marvell_nfc *nfc) if (ret) return ret; - nfc->dma_chan = dma_request_slave_channel(nfc->dev, "data"); - if (!nfc->dma_chan) { - dev_err(nfc->dev, - "Unable to request data DMA channel\n"); - return -ENODEV; + nfc->dma_chan = dma_request_chan(nfc->dev, "data"); + if (IS_ERR(nfc->dma_chan)) { + ret = PTR_ERR(nfc->dma_chan); + nfc->dma_chan = NULL; + if (ret != -EPROBE_DEFER) + dev_err(nfc->dev, "DMA channel request failed: %d\n", + ret); + return ret; } r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) - return -ENXIO; + if (!r) { + ret = -ENXIO; + goto release_channel; + } config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; @@ -2763,7 +2768,7 @@ static int marvell_nfc_init_dma(struct marvell_nfc *nfc) ret = dmaengine_slave_config(nfc->dma_chan, &config); if (ret < 0) { dev_err(nfc->dev, "Failed to configure DMA channel\n"); - return ret; + goto release_channel; } /* @@ -2773,12 +2778,20 @@ static int marvell_nfc_init_dma(struct marvell_nfc *nfc) * the provided buffer. */ nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA); - if (!nfc->dma_buf) - return -ENOMEM; + if (!nfc->dma_buf) { + ret = -ENOMEM; + goto release_channel; + } nfc->use_dma = true; return 0; + +release_channel: + dma_release_channel(nfc->dma_chan); + nfc->dma_chan = NULL; + + return ret; } static void marvell_nfc_reset(struct marvell_nfc *nfc) @@ -2920,10 +2933,13 @@ static int marvell_nfc_probe(struct platform_device *pdev) ret = marvell_nand_chips_init(dev, nfc); if (ret) - goto unprepare_reg_clk; + goto release_dma; return 0; +release_dma: + if (nfc->use_dma) + dma_release_channel(nfc->dma_chan); unprepare_reg_clk: clk_disable_unprepare(nfc->reg_clk); unprepare_core_clk: diff --git a/drivers/mtd/nand/raw/meson_nand.c b/drivers/mtd/nand/raw/meson_nand.c index 9f17b5b8efbf..f6fb5c0e6255 100644 --- a/drivers/mtd/nand/raw/meson_nand.c +++ b/drivers/mtd/nand/raw/meson_nand.c @@ -118,7 +118,7 @@ struct meson_nfc_nand_chip { u8 *data_buf; __le64 *info_buf; u32 nsels; - u8 sels[0]; + u8 sels[]; }; struct meson_nand_ecc { diff --git a/drivers/mtd/nand/raw/mtk_nand.c b/drivers/mtd/nand/raw/mtk_nand.c index b8305e39ab51..ef149e8b26d0 100644 --- a/drivers/mtd/nand/raw/mtk_nand.c +++ b/drivers/mtd/nand/raw/mtk_nand.c @@ -131,7 +131,7 @@ struct mtk_nfc_nand_chip { u32 spare_per_sector; int nsels; - u8 sels[0]; + u8 sels[]; /* nothing after this field */ }; diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index f64e3b6605c6..c24e5e2ba130 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -683,7 +683,12 @@ int nand_soft_waitrdy(struct nand_chip *chip, unsigned long timeout_ms) if (ret) return ret; - timeout_ms = jiffies + msecs_to_jiffies(timeout_ms); + /* + * +1 below is necessary because if we are now in the last fraction + * of jiffy and msecs_to_jiffies is 1 then we will wait only that + * small jiffy fraction - possibly leading to false timeout + */ + timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1; do { ret = nand_read_data_op(chip, &status, sizeof(status), true); if (ret) @@ -4321,16 +4326,22 @@ static int nand_block_markbad(struct mtd_info *mtd, loff_t ofs) /** * nand_suspend - [MTD Interface] Suspend the NAND flash * @mtd: MTD device structure + * + * Returns 0 for success or negative error code otherwise. */ static int nand_suspend(struct mtd_info *mtd) { struct nand_chip *chip = mtd_to_nand(mtd); + int ret = 0; mutex_lock(&chip->lock); - chip->suspended = 1; + if (chip->suspend) + ret = chip->suspend(chip); + if (!ret) + chip->suspended = 1; mutex_unlock(&chip->lock); - return 0; + return ret; } /** @@ -4342,11 +4353,14 @@ static void nand_resume(struct mtd_info *mtd) struct nand_chip *chip = mtd_to_nand(mtd); mutex_lock(&chip->lock); - if (chip->suspended) + if (chip->suspended) { + if (chip->resume) + chip->resume(chip); chip->suspended = 0; - else + } else { pr_err("%s called for a chip which is not in suspended state\n", __func__); + } mutex_unlock(&chip->lock); } @@ -4360,6 +4374,38 @@ static void nand_shutdown(struct mtd_info *mtd) nand_suspend(mtd); } +/** + * nand_lock - [MTD Interface] Lock the NAND flash + * @mtd: MTD device structure + * @ofs: offset byte address + * @len: number of bytes to lock (must be a multiple of block/page size) + */ +static int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + + if (!chip->lock_area) + return -ENOTSUPP; + + return chip->lock_area(chip, ofs, len); +} + +/** + * nand_unlock - [MTD Interface] Unlock the NAND flash + * @mtd: MTD device structure + * @ofs: offset byte address + * @len: number of bytes to unlock (must be a multiple of block/page size) + */ +static int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + + if (!chip->unlock_area) + return -ENOTSUPP; + + return chip->unlock_area(chip, ofs, len); +} + /* Set default functions */ static void nand_set_defaults(struct nand_chip *chip) { @@ -5591,8 +5637,7 @@ static int nand_scan_tail(struct nand_chip *chip) } if (!ecc->read_page) ecc->read_page = nand_read_page_hwecc_oob_first; - /* fall through */ - + fallthrough; case NAND_ECC_HW: /* Use standard hwecc read page function? */ if (!ecc->read_page) @@ -5611,8 +5656,7 @@ static int nand_scan_tail(struct nand_chip *chip) ecc->read_subpage = nand_read_subpage; if (!ecc->write_subpage && ecc->hwctl && ecc->calculate) ecc->write_subpage = nand_write_subpage_hwecc; - /* fall through */ - + fallthrough; case NAND_ECC_HW_SYNDROME: if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) && (!ecc->read_page || @@ -5649,8 +5693,7 @@ static int nand_scan_tail(struct nand_chip *chip) ecc->size, mtd->writesize); ecc->mode = NAND_ECC_SOFT; ecc->algo = NAND_ECC_HAMMING; - /* fall through */ - + fallthrough; case NAND_ECC_SOFT: ret = nand_set_ecc_soft_ops(chip); if (ret) { @@ -5786,8 +5829,8 @@ static int nand_scan_tail(struct nand_chip *chip) mtd->_read_oob = nand_read_oob; mtd->_write_oob = nand_write_oob; mtd->_sync = nand_sync; - mtd->_lock = NULL; - mtd->_unlock = NULL; + mtd->_lock = nand_lock; + mtd->_unlock = nand_unlock; mtd->_suspend = nand_suspend; mtd->_resume = nand_resume; mtd->_reboot = nand_shutdown; @@ -5907,6 +5950,8 @@ void nand_cleanup(struct nand_chip *chip) chip->ecc.algo == NAND_ECC_BCH) nand_bch_free((struct nand_bch_control *)chip->ecc.priv); + nanddev_cleanup(&chip->base); + /* Free bad block table memory */ kfree(chip->bbt); kfree(chip->data_buf); diff --git a/drivers/mtd/nand/raw/nand_hynix.c b/drivers/mtd/nand/raw/nand_hynix.c index 194e4227aefe..7caedaa5b9e5 100644 --- a/drivers/mtd/nand/raw/nand_hynix.c +++ b/drivers/mtd/nand/raw/nand_hynix.c @@ -26,7 +26,7 @@ struct hynix_read_retry { int nregs; const u8 *regs; - u8 values[0]; + u8 values[]; }; /** diff --git a/drivers/mtd/nand/raw/nand_legacy.c b/drivers/mtd/nand/raw/nand_legacy.c index f2526ec616a6..f91e92e1b972 100644 --- a/drivers/mtd/nand/raw/nand_legacy.c +++ b/drivers/mtd/nand/raw/nand_legacy.c @@ -331,8 +331,7 @@ static void nand_command(struct nand_chip *chip, unsigned int command, */ if (column == -1 && page_addr == -1) return; - /* fall through */ - + fallthrough; default: /* * If we don't have access to the busy pin, we apply the given @@ -483,8 +482,7 @@ static void nand_command_lp(struct nand_chip *chip, unsigned int command, NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE); chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); - - /* fall through - This applies to read commands */ + fallthrough; /* This applies to read commands */ default: /* * If we don't have access to the busy pin, we apply the given diff --git a/drivers/mtd/nand/raw/nand_macronix.c b/drivers/mtd/nand/raw/nand_macronix.c index 3ff7ce00cbdb..09c254c97b5c 100644 --- a/drivers/mtd/nand/raw/nand_macronix.c +++ b/drivers/mtd/nand/raw/nand_macronix.c @@ -6,11 +6,31 @@ * Author: Boris Brezillon */ +#include "linux/delay.h" #include "internals.h" #define MACRONIX_READ_RETRY_BIT BIT(0) #define MACRONIX_NUM_READ_RETRY_MODES 6 +#define ONFI_FEATURE_ADDR_MXIC_PROTECTION 0xA0 +#define MXIC_BLOCK_PROTECTION_ALL_LOCK 0x38 +#define MXIC_BLOCK_PROTECTION_ALL_UNLOCK 0x0 + +#define ONFI_FEATURE_ADDR_MXIC_RANDOMIZER 0xB0 +#define MACRONIX_RANDOMIZER_BIT BIT(1) +#define MACRONIX_RANDOMIZER_ENPGM BIT(0) +#define MACRONIX_RANDOMIZER_RANDEN BIT(1) +#define MACRONIX_RANDOMIZER_RANDOPT BIT(2) +#define MACRONIX_RANDOMIZER_MODE_ENTER \ + (MACRONIX_RANDOMIZER_ENPGM | \ + MACRONIX_RANDOMIZER_RANDEN | \ + MACRONIX_RANDOMIZER_RANDOPT) +#define MACRONIX_RANDOMIZER_MODE_EXIT \ + (MACRONIX_RANDOMIZER_RANDEN | \ + MACRONIX_RANDOMIZER_RANDOPT) + +#define MXIC_CMD_POWER_DOWN 0xB9 + struct nand_onfi_vendor_macronix { u8 reserved; u8 reliability_func; @@ -29,15 +49,83 @@ static int macronix_nand_setup_read_retry(struct nand_chip *chip, int mode) return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature); } +static int macronix_nand_randomizer_check_enable(struct nand_chip *chip) +{ + u8 feature[ONFI_SUBFEATURE_PARAM_LEN]; + int ret; + + ret = nand_get_features(chip, ONFI_FEATURE_ADDR_MXIC_RANDOMIZER, + feature); + if (ret < 0) + return ret; + + if (feature[0]) + return feature[0]; + + feature[0] = MACRONIX_RANDOMIZER_MODE_ENTER; + ret = nand_set_features(chip, ONFI_FEATURE_ADDR_MXIC_RANDOMIZER, + feature); + if (ret < 0) + return ret; + + /* RANDEN and RANDOPT OTP bits are programmed */ + feature[0] = 0x0; + ret = nand_prog_page_op(chip, 0, 0, feature, 1); + if (ret < 0) + return ret; + + ret = nand_get_features(chip, ONFI_FEATURE_ADDR_MXIC_RANDOMIZER, + feature); + if (ret < 0) + return ret; + + feature[0] &= MACRONIX_RANDOMIZER_MODE_EXIT; + ret = nand_set_features(chip, ONFI_FEATURE_ADDR_MXIC_RANDOMIZER, + feature); + if (ret < 0) + return ret; + + return 0; +} + static void macronix_nand_onfi_init(struct nand_chip *chip) { struct nand_parameters *p = &chip->parameters; struct nand_onfi_vendor_macronix *mxic; + struct device_node *dn = nand_get_flash_node(chip); + int rand_otp = 0; + int ret; if (!p->onfi) return; + if (of_find_property(dn, "mxic,enable-randomizer-otp", NULL)) + rand_otp = 1; + mxic = (struct nand_onfi_vendor_macronix *)p->onfi->vendor; + /* Subpage write is prohibited in randomizer operatoin */ + if (rand_otp && chip->options & NAND_NO_SUBPAGE_WRITE && + mxic->reliability_func & MACRONIX_RANDOMIZER_BIT) { + if (p->supports_set_get_features) { + bitmap_set(p->set_feature_list, + ONFI_FEATURE_ADDR_MXIC_RANDOMIZER, 1); + bitmap_set(p->get_feature_list, + ONFI_FEATURE_ADDR_MXIC_RANDOMIZER, 1); + ret = macronix_nand_randomizer_check_enable(chip); + if (ret < 0) { + bitmap_clear(p->set_feature_list, + ONFI_FEATURE_ADDR_MXIC_RANDOMIZER, + 1); + bitmap_clear(p->get_feature_list, + ONFI_FEATURE_ADDR_MXIC_RANDOMIZER, + 1); + pr_info("Macronix NAND randomizer failed\n"); + } else { + pr_info("Macronix NAND randomizer enabled\n"); + } + } + } + if ((mxic->reliability_func & MACRONIX_READ_RETRY_BIT) == 0) return; @@ -91,6 +179,143 @@ static void macronix_nand_fix_broken_get_timings(struct nand_chip *chip) ONFI_FEATURE_ADDR_TIMING_MODE, 1); } +/* + * Macronix NAND supports Block Protection by Protectoin(PT) pin; + * active high at power-on which protects the entire chip even the #WP is + * disabled. Lock/unlock protection area can be partition according to + * protection bits, i.e. upper 1/2 locked, upper 1/4 locked and so on. + */ +static int mxic_nand_lock(struct nand_chip *chip, loff_t ofs, uint64_t len) +{ + u8 feature[ONFI_SUBFEATURE_PARAM_LEN]; + int ret; + + feature[0] = MXIC_BLOCK_PROTECTION_ALL_LOCK; + nand_select_target(chip, 0); + ret = nand_set_features(chip, ONFI_FEATURE_ADDR_MXIC_PROTECTION, + feature); + nand_deselect_target(chip); + if (ret) + pr_err("%s all blocks failed\n", __func__); + + return ret; +} + +static int mxic_nand_unlock(struct nand_chip *chip, loff_t ofs, uint64_t len) +{ + u8 feature[ONFI_SUBFEATURE_PARAM_LEN]; + int ret; + + feature[0] = MXIC_BLOCK_PROTECTION_ALL_UNLOCK; + nand_select_target(chip, 0); + ret = nand_set_features(chip, ONFI_FEATURE_ADDR_MXIC_PROTECTION, + feature); + nand_deselect_target(chip); + if (ret) + pr_err("%s all blocks failed\n", __func__); + + return ret; +} + +static void macronix_nand_block_protection_support(struct nand_chip *chip) +{ + u8 feature[ONFI_SUBFEATURE_PARAM_LEN]; + int ret; + + bitmap_set(chip->parameters.get_feature_list, + ONFI_FEATURE_ADDR_MXIC_PROTECTION, 1); + + feature[0] = MXIC_BLOCK_PROTECTION_ALL_UNLOCK; + nand_select_target(chip, 0); + ret = nand_get_features(chip, ONFI_FEATURE_ADDR_MXIC_PROTECTION, + feature); + nand_deselect_target(chip); + if (ret || feature[0] != MXIC_BLOCK_PROTECTION_ALL_LOCK) { + if (ret) + pr_err("Block protection check failed\n"); + + bitmap_clear(chip->parameters.get_feature_list, + ONFI_FEATURE_ADDR_MXIC_PROTECTION, 1); + return; + } + + bitmap_set(chip->parameters.set_feature_list, + ONFI_FEATURE_ADDR_MXIC_PROTECTION, 1); + + chip->lock_area = mxic_nand_lock; + chip->unlock_area = mxic_nand_unlock; +} + +static int nand_power_down_op(struct nand_chip *chip) +{ + int ret; + + if (nand_has_exec_op(chip)) { + struct nand_op_instr instrs[] = { + NAND_OP_CMD(MXIC_CMD_POWER_DOWN, 0), + }; + + struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); + + ret = nand_exec_op(chip, &op); + if (ret) + return ret; + + } else { + chip->legacy.cmdfunc(chip, MXIC_CMD_POWER_DOWN, -1, -1); + } + + return 0; +} + +static int mxic_nand_suspend(struct nand_chip *chip) +{ + int ret; + + nand_select_target(chip, 0); + ret = nand_power_down_op(chip); + if (ret < 0) + pr_err("Suspending MXIC NAND chip failed (%d)\n", ret); + nand_deselect_target(chip); + + return ret; +} + +static void mxic_nand_resume(struct nand_chip *chip) +{ + /* + * Toggle #CS pin to resume NAND device and don't care + * of the others CLE, #WE, #RE pins status. + * A NAND controller ensure it is able to assert/de-assert #CS + * by sending any byte over the NAND bus. + * i.e., + * NAND power down command or reset command w/o R/B# status checking. + */ + nand_select_target(chip, 0); + nand_power_down_op(chip); + /* The minimum of a recovery time tRDP is 35 us */ + usleep_range(35, 100); + nand_deselect_target(chip); +} + +static void macronix_nand_deep_power_down_support(struct nand_chip *chip) +{ + int i; + static const char * const deep_power_down_dev[] = { + "MX30UF1G28AD", + "MX30UF2G28AD", + "MX30UF4G28AD", + }; + + i = match_string(deep_power_down_dev, ARRAY_SIZE(deep_power_down_dev), + chip->parameters.model); + if (i < 0) + return; + + chip->suspend = mxic_nand_suspend; + chip->resume = mxic_nand_resume; +} + static int macronix_nand_init(struct nand_chip *chip) { if (nand_is_slc(chip)) @@ -98,6 +323,8 @@ static int macronix_nand_init(struct nand_chip *chip) macronix_nand_fix_broken_get_timings(chip); macronix_nand_onfi_init(chip); + macronix_nand_block_protection_support(chip); + macronix_nand_deep_power_down_support(chip); return 0; } diff --git a/drivers/mtd/nand/raw/nand_toshiba.c b/drivers/mtd/nand/raw/nand_toshiba.c index 9c03fbb1f47d..f3dcd695b5db 100644 --- a/drivers/mtd/nand/raw/nand_toshiba.c +++ b/drivers/mtd/nand/raw/nand_toshiba.c @@ -14,14 +14,68 @@ /* Recommended to rewrite for BENAND */ #define TOSHIBA_NAND_STATUS_REWRITE_RECOMMENDED BIT(3) +/* ECC Status Read Command for BENAND */ +#define TOSHIBA_NAND_CMD_ECC_STATUS_READ 0x7A + +/* ECC Status Mask for BENAND */ +#define TOSHIBA_NAND_ECC_STATUS_MASK 0x0F + +/* Uncorrectable Error for BENAND */ +#define TOSHIBA_NAND_ECC_STATUS_UNCORR 0x0F + +/* Max ECC Steps for BENAND */ +#define TOSHIBA_NAND_MAX_ECC_STEPS 8 + +static int toshiba_nand_benand_read_eccstatus_op(struct nand_chip *chip, + u8 *buf) +{ + u8 *ecc_status = buf; + + if (nand_has_exec_op(chip)) { + const struct nand_sdr_timings *sdr = + nand_get_sdr_timings(&chip->data_interface); + struct nand_op_instr instrs[] = { + NAND_OP_CMD(TOSHIBA_NAND_CMD_ECC_STATUS_READ, + PSEC_TO_NSEC(sdr->tADL_min)), + NAND_OP_8BIT_DATA_IN(chip->ecc.steps, ecc_status, 0), + }; + struct nand_operation op = NAND_OPERATION(chip->cur_cs, instrs); + + return nand_exec_op(chip, &op); + } + + return -ENOTSUPP; +} + static int toshiba_nand_benand_eccstatus(struct nand_chip *chip) { struct mtd_info *mtd = nand_to_mtd(chip); int ret; unsigned int max_bitflips = 0; - u8 status; + u8 status, ecc_status[TOSHIBA_NAND_MAX_ECC_STEPS]; /* Check Status */ + ret = toshiba_nand_benand_read_eccstatus_op(chip, ecc_status); + if (!ret) { + unsigned int i, bitflips = 0; + + for (i = 0; i < chip->ecc.steps; i++) { + bitflips = ecc_status[i] & TOSHIBA_NAND_ECC_STATUS_MASK; + if (bitflips == TOSHIBA_NAND_ECC_STATUS_UNCORR) { + mtd->ecc_stats.failed++; + } else { + mtd->ecc_stats.corrected += bitflips; + max_bitflips = max(max_bitflips, bitflips); + } + } + + return max_bitflips; + } + + /* + * Fallback to regular status check if + * toshiba_nand_benand_read_eccstatus_op() failed. + */ ret = nand_status_op(chip, &status); if (ret) return ret; @@ -108,7 +162,7 @@ static void toshiba_nand_decode_id(struct nand_chip *chip) */ if (chip->id.len >= 6 && nand_is_slc(chip) && (chip->id.data[5] & 0x7) == 0x6 /* 24nm */ && - !(chip->id.data[4] & 0x80) /* !BENAND */) { + !(chip->id.data[4] & TOSHIBA_NAND_ID4_IS_BENAND) /* !BENAND */) { memorg->oobsize = 32 * memorg->pagesize >> 9; mtd->oobsize = memorg->oobsize; } diff --git a/drivers/mtd/nand/raw/nandsim.c b/drivers/mtd/nand/raw/nandsim.c index 9a70754a61ef..1de03bb34e84 100644 --- a/drivers/mtd/nand/raw/nandsim.c +++ b/drivers/mtd/nand/raw/nandsim.c @@ -2251,10 +2251,10 @@ static int __init ns_init_module(void) switch (bbt) { case 2: chip->bbt_options |= NAND_BBT_NO_OOB; - /* fall through */ + fallthrough; case 1: chip->bbt_options |= NAND_BBT_USE_FLASH; - /* fall through */ + fallthrough; case 0: break; default: diff --git a/drivers/mtd/nand/raw/omap_elm.c b/drivers/mtd/nand/raw/omap_elm.c index 5502ffbdd1e6..3fa0e2cbbe53 100644 --- a/drivers/mtd/nand/raw/omap_elm.c +++ b/drivers/mtd/nand/raw/omap_elm.c @@ -455,13 +455,13 @@ static int elm_context_save(struct elm_info *info) ELM_SYNDROME_FRAGMENT_5 + offset); regs->elm_syndrome_fragment_4[i] = elm_read_reg(info, ELM_SYNDROME_FRAGMENT_4 + offset); - /* fall through */ + fallthrough; case BCH8_ECC: regs->elm_syndrome_fragment_3[i] = elm_read_reg(info, ELM_SYNDROME_FRAGMENT_3 + offset); regs->elm_syndrome_fragment_2[i] = elm_read_reg(info, ELM_SYNDROME_FRAGMENT_2 + offset); - /* fall through */ + fallthrough; case BCH4_ECC: regs->elm_syndrome_fragment_1[i] = elm_read_reg(info, ELM_SYNDROME_FRAGMENT_1 + offset); @@ -503,13 +503,13 @@ static int elm_context_restore(struct elm_info *info) regs->elm_syndrome_fragment_5[i]); elm_write_reg(info, ELM_SYNDROME_FRAGMENT_4 + offset, regs->elm_syndrome_fragment_4[i]); - /* fall through */ + fallthrough; case BCH8_ECC: elm_write_reg(info, ELM_SYNDROME_FRAGMENT_3 + offset, regs->elm_syndrome_fragment_3[i]); elm_write_reg(info, ELM_SYNDROME_FRAGMENT_2 + offset, regs->elm_syndrome_fragment_2[i]); - /* fall through */ + fallthrough; case BCH4_ECC: elm_write_reg(info, ELM_SYNDROME_FRAGMENT_1 + offset, regs->elm_syndrome_fragment_1[i]); diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c index 7bb9a7e8e1e7..5b11c7061497 100644 --- a/drivers/mtd/nand/raw/qcom_nandc.c +++ b/drivers/mtd/nand/raw/qcom_nandc.c @@ -2628,6 +2628,29 @@ static const struct nand_controller_ops qcom_nandc_ops = { .attach_chip = qcom_nand_attach_chip, }; +static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc) +{ + if (nandc->props->is_bam) { + if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma)) + dma_unmap_single(nandc->dev, nandc->reg_read_dma, + MAX_REG_RD * + sizeof(*nandc->reg_read_buf), + DMA_FROM_DEVICE); + + if (nandc->tx_chan) + dma_release_channel(nandc->tx_chan); + + if (nandc->rx_chan) + dma_release_channel(nandc->rx_chan); + + if (nandc->cmd_chan) + dma_release_channel(nandc->cmd_chan); + } else { + if (nandc->chan) + dma_release_channel(nandc->chan); + } +} + static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) { int ret; @@ -2673,22 +2696,37 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) return -EIO; } - nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx"); - if (!nandc->tx_chan) { - dev_err(nandc->dev, "failed to request tx channel\n"); - return -ENODEV; + nandc->tx_chan = dma_request_chan(nandc->dev, "tx"); + if (IS_ERR(nandc->tx_chan)) { + ret = PTR_ERR(nandc->tx_chan); + nandc->tx_chan = NULL; + if (ret != -EPROBE_DEFER) + dev_err(nandc->dev, + "tx DMA channel request failed: %d\n", + ret); + goto unalloc; } - nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx"); - if (!nandc->rx_chan) { - dev_err(nandc->dev, "failed to request rx channel\n"); - return -ENODEV; + nandc->rx_chan = dma_request_chan(nandc->dev, "rx"); + if (IS_ERR(nandc->rx_chan)) { + ret = PTR_ERR(nandc->rx_chan); + nandc->rx_chan = NULL; + if (ret != -EPROBE_DEFER) + dev_err(nandc->dev, + "rx DMA channel request failed: %d\n", + ret); + goto unalloc; } - nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd"); - if (!nandc->cmd_chan) { - dev_err(nandc->dev, "failed to request cmd channel\n"); - return -ENODEV; + nandc->cmd_chan = dma_request_chan(nandc->dev, "cmd"); + if (IS_ERR(nandc->cmd_chan)) { + ret = PTR_ERR(nandc->cmd_chan); + nandc->cmd_chan = NULL; + if (ret != -EPROBE_DEFER) + dev_err(nandc->dev, + "cmd DMA channel request failed: %d\n", + ret); + goto unalloc; } /* @@ -2702,14 +2740,19 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) if (!nandc->bam_txn) { dev_err(nandc->dev, "failed to allocate bam transaction\n"); - return -ENOMEM; + ret = -ENOMEM; + goto unalloc; } } else { - nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx"); - if (!nandc->chan) { - dev_err(nandc->dev, - "failed to request slave channel\n"); - return -ENODEV; + nandc->chan = dma_request_chan(nandc->dev, "rxtx"); + if (IS_ERR(nandc->chan)) { + ret = PTR_ERR(nandc->chan); + nandc->chan = NULL; + if (ret != -EPROBE_DEFER) + dev_err(nandc->dev, + "rxtx DMA channel request failed: %d\n", + ret); + return ret; } } @@ -2720,29 +2763,9 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc) nandc->controller.ops = &qcom_nandc_ops; return 0; -} - -static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc) -{ - if (nandc->props->is_bam) { - if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma)) - dma_unmap_single(nandc->dev, nandc->reg_read_dma, - MAX_REG_RD * - sizeof(*nandc->reg_read_buf), - DMA_FROM_DEVICE); - - if (nandc->tx_chan) - dma_release_channel(nandc->tx_chan); - - if (nandc->rx_chan) - dma_release_channel(nandc->rx_chan); - - if (nandc->cmd_chan) - dma_release_channel(nandc->cmd_chan); - } else { - if (nandc->chan) - dma_release_channel(nandc->chan); - } +unalloc: + qcom_nandc_unalloc(nandc); + return ret; } /* one time setup of a few nand controller registers */ diff --git a/drivers/mtd/nand/raw/stm32_fmc2_nand.c b/drivers/mtd/nand/raw/stm32_fmc2_nand.c index 3ba73f18841f..b6d45cd911ae 100644 --- a/drivers/mtd/nand/raw/stm32_fmc2_nand.c +++ b/drivers/mtd/nand/raw/stm32_fmc2_nand.c @@ -1606,15 +1606,36 @@ static int stm32_fmc2_setup_interface(struct nand_chip *chip, int chipnr, /* DMA configuration */ static int stm32_fmc2_dma_setup(struct stm32_fmc2_nfc *fmc2) { - int ret; + int ret = 0; - fmc2->dma_tx_ch = dma_request_slave_channel(fmc2->dev, "tx"); - fmc2->dma_rx_ch = dma_request_slave_channel(fmc2->dev, "rx"); - fmc2->dma_ecc_ch = dma_request_slave_channel(fmc2->dev, "ecc"); + fmc2->dma_tx_ch = dma_request_chan(fmc2->dev, "tx"); + if (IS_ERR(fmc2->dma_tx_ch)) { + ret = PTR_ERR(fmc2->dma_tx_ch); + if (ret != -ENODEV) + dev_err(fmc2->dev, + "failed to request tx DMA channel: %d\n", ret); + fmc2->dma_tx_ch = NULL; + goto err_dma; + } - if (!fmc2->dma_tx_ch || !fmc2->dma_rx_ch || !fmc2->dma_ecc_ch) { - dev_warn(fmc2->dev, "DMAs not defined in the device tree, polling mode is used\n"); - return 0; + fmc2->dma_rx_ch = dma_request_chan(fmc2->dev, "rx"); + if (IS_ERR(fmc2->dma_rx_ch)) { + ret = PTR_ERR(fmc2->dma_rx_ch); + if (ret != -ENODEV) + dev_err(fmc2->dev, + "failed to request rx DMA channel: %d\n", ret); + fmc2->dma_rx_ch = NULL; + goto err_dma; + } + + fmc2->dma_ecc_ch = dma_request_chan(fmc2->dev, "ecc"); + if (IS_ERR(fmc2->dma_ecc_ch)) { + ret = PTR_ERR(fmc2->dma_ecc_ch); + if (ret != -ENODEV) + dev_err(fmc2->dev, + "failed to request ecc DMA channel: %d\n", ret); + fmc2->dma_ecc_ch = NULL; + goto err_dma; } ret = sg_alloc_table(&fmc2->dma_ecc_sg, FMC2_MAX_SG, GFP_KERNEL); @@ -1635,6 +1656,15 @@ static int stm32_fmc2_dma_setup(struct stm32_fmc2_nfc *fmc2) init_completion(&fmc2->dma_ecc_complete); return 0; + +err_dma: + if (ret == -ENODEV) { + dev_warn(fmc2->dev, + "DMAs not defined in the DT, polling mode is used\n"); + ret = 0; + } + + return ret; } /* NAND callbacks setup */ diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c index 37a4ac0dd85b..5f3e40b79fb1 100644 --- a/drivers/mtd/nand/raw/sunxi_nand.c +++ b/drivers/mtd/nand/raw/sunxi_nand.c @@ -195,7 +195,7 @@ struct sunxi_nand_chip { u32 timing_cfg; u32 timing_ctl; int nsels; - struct sunxi_nand_chip_sel sels[0]; + struct sunxi_nand_chip_sel sels[]; }; static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) @@ -2123,8 +2123,16 @@ static int sunxi_nfc_probe(struct platform_device *pdev) if (ret) goto out_ahb_reset_reassert; - nfc->dmac = dma_request_slave_channel(dev, "rxtx"); - if (nfc->dmac) { + nfc->dmac = dma_request_chan(dev, "rxtx"); + if (IS_ERR(nfc->dmac)) { + ret = PTR_ERR(nfc->dmac); + if (ret == -EPROBE_DEFER) + goto out_ahb_reset_reassert; + + /* Ignore errors to fall back to PIO mode */ + dev_warn(dev, "failed to request rxtx DMA channel: %d\n", ret); + nfc->dmac = NULL; + } else { struct dma_slave_config dmac_cfg = { }; dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data; @@ -2138,9 +2146,6 @@ static int sunxi_nfc_probe(struct platform_device *pdev) if (nfc->caps->extra_mbus_conf) writel(readl(nfc->regs + NFC_REG_CTL) | NFC_DMA_TYPE_NORMAL, nfc->regs + NFC_REG_CTL); - - } else { - dev_warn(dev, "failed to request rxtx DMA channel\n"); } platform_set_drvdata(pdev, nfc); diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index 89f6beefb01c..b6bb358b96ce 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -370,10 +371,11 @@ static int spinand_wait(struct spinand_device *spinand, u8 *s) return status & STATUS_BUSY ? -ETIMEDOUT : 0; } -static int spinand_read_id_op(struct spinand_device *spinand, u8 *buf) +static int spinand_read_id_op(struct spinand_device *spinand, u8 naddr, + u8 ndummy, u8 *buf) { - struct spi_mem_op op = SPINAND_READID_OP(0, spinand->scratchbuf, - SPINAND_MAX_ID_LEN); + struct spi_mem_op op = SPINAND_READID_OP( + naddr, ndummy, spinand->scratchbuf, SPINAND_MAX_ID_LEN); int ret; ret = spi_mem_exec_op(spinand->spimem, &op); @@ -568,18 +570,18 @@ static int spinand_mtd_write(struct mtd_info *mtd, loff_t to, static bool spinand_isbad(struct nand_device *nand, const struct nand_pos *pos) { struct spinand_device *spinand = nand_to_spinand(nand); + u8 marker[2] = { }; struct nand_page_io_req req = { .pos = *pos, - .ooblen = 2, + .ooblen = sizeof(marker), .ooboffs = 0, - .oobbuf.in = spinand->oobbuf, + .oobbuf.in = marker, .mode = MTD_OPS_RAW, }; - memset(spinand->oobbuf, 0, 2); spinand_select_target(spinand, pos->target); spinand_read_page(spinand, &req, false); - if (spinand->oobbuf[0] != 0xff || spinand->oobbuf[1] != 0xff) + if (marker[0] != 0xff || marker[1] != 0xff) return true; return false; @@ -603,15 +605,16 @@ static int spinand_mtd_block_isbad(struct mtd_info *mtd, loff_t offs) static int spinand_markbad(struct nand_device *nand, const struct nand_pos *pos) { struct spinand_device *spinand = nand_to_spinand(nand); + u8 marker[2] = { }; struct nand_page_io_req req = { .pos = *pos, .ooboffs = 0, - .ooblen = 2, - .oobbuf.out = spinand->oobbuf, + .ooblen = sizeof(marker), + .oobbuf.out = marker, + .mode = MTD_OPS_RAW, }; int ret; - /* Erase block before marking it bad. */ ret = spinand_select_target(spinand, pos->target); if (ret) return ret; @@ -620,9 +623,6 @@ static int spinand_markbad(struct nand_device *nand, const struct nand_pos *pos) if (ret) return ret; - spinand_erase_op(spinand, pos); - - memset(spinand->oobbuf, 0, 2); return spinand_write_page(spinand, &req); } @@ -762,24 +762,62 @@ static const struct spinand_manufacturer *spinand_manufacturers[] = { &winbond_spinand_manufacturer, }; -static int spinand_manufacturer_detect(struct spinand_device *spinand) +static int spinand_manufacturer_match(struct spinand_device *spinand, + enum spinand_readid_method rdid_method) { + u8 *id = spinand->id.data; unsigned int i; int ret; for (i = 0; i < ARRAY_SIZE(spinand_manufacturers); i++) { - ret = spinand_manufacturers[i]->ops->detect(spinand); - if (ret > 0) { - spinand->manufacturer = spinand_manufacturers[i]; - return 0; - } else if (ret < 0) { - return ret; - } - } + const struct spinand_manufacturer *manufacturer = + spinand_manufacturers[i]; + if (id[0] != manufacturer->id) + continue; + + ret = spinand_match_and_init(spinand, + manufacturer->chips, + manufacturer->nchips, + rdid_method); + if (ret < 0) + continue; + + spinand->manufacturer = manufacturer; + return 0; + } return -ENOTSUPP; } +static int spinand_id_detect(struct spinand_device *spinand) +{ + u8 *id = spinand->id.data; + int ret; + + ret = spinand_read_id_op(spinand, 0, 0, id); + if (ret) + return ret; + ret = spinand_manufacturer_match(spinand, SPINAND_READID_METHOD_OPCODE); + if (!ret) + return 0; + + ret = spinand_read_id_op(spinand, 1, 0, id); + if (ret) + return ret; + ret = spinand_manufacturer_match(spinand, + SPINAND_READID_METHOD_OPCODE_ADDR); + if (!ret) + return 0; + + ret = spinand_read_id_op(spinand, 0, 1, id); + if (ret) + return ret; + ret = spinand_manufacturer_match(spinand, + SPINAND_READID_METHOD_OPCODE_DUMMY); + + return ret; +} + static int spinand_manufacturer_init(struct spinand_device *spinand) { if (spinand->manufacturer->ops->init) @@ -835,9 +873,9 @@ spinand_select_op_variant(struct spinand_device *spinand, * @spinand: SPI NAND object * @table: SPI NAND device description table * @table_size: size of the device description table + * @rdid_method: read id method to match * - * Should be used by SPI NAND manufacturer drivers when they want to find a - * match between a device ID retrieved through the READ_ID command and an + * Match between a device ID retrieved through the READ_ID command and an * entry in the SPI NAND description table. If a match is found, the spinand * object will be initialized with information provided by the matching * spinand_info entry. @@ -846,8 +884,10 @@ spinand_select_op_variant(struct spinand_device *spinand, */ int spinand_match_and_init(struct spinand_device *spinand, const struct spinand_info *table, - unsigned int table_size, u16 devid) + unsigned int table_size, + enum spinand_readid_method rdid_method) { + u8 *id = spinand->id.data; struct nand_device *nand = spinand_to_nand(spinand); unsigned int i; @@ -855,13 +895,17 @@ int spinand_match_and_init(struct spinand_device *spinand, const struct spinand_info *info = &table[i]; const struct spi_mem_op *op; - if (devid != info->devid) + if (rdid_method != info->devid.method) + continue; + + if (memcmp(id + 1, info->devid.id, info->devid.len)) continue; nand->memorg = table[i].memorg; nand->eccreq = table[i].eccreq; spinand->eccinfo = table[i].eccinfo; spinand->flags = table[i].flags; + spinand->id.len = 1 + table[i].devid.len; spinand->select_target = table[i].select_target; op = spinand_select_op_variant(spinand, @@ -898,13 +942,7 @@ static int spinand_detect(struct spinand_device *spinand) if (ret) return ret; - ret = spinand_read_id_op(spinand, spinand->id.data); - if (ret) - return ret; - - spinand->id.len = SPINAND_MAX_ID_LEN; - - ret = spinand_manufacturer_detect(spinand); + ret = spinand_id_detect(spinand); if (ret) { dev_err(dev, "unknown raw ID %*phN\n", SPINAND_MAX_ID_LEN, spinand->id.data); diff --git a/drivers/mtd/nand/spi/gigadevice.c b/drivers/mtd/nand/spi/gigadevice.c index e99d425aa93f..d219c970042a 100644 --- a/drivers/mtd/nand/spi/gigadevice.c +++ b/drivers/mtd/nand/spi/gigadevice.c @@ -195,7 +195,8 @@ static int gd5fxgq4ufxxg_ecc_get_status(struct spinand_device *spinand, } static const struct spinand_info gigadevice_spinand_table[] = { - SPINAND_INFO("GD5F1GQ4xA", 0xF1, + SPINAND_INFO("GD5F1GQ4xA", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0xf1), NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -204,7 +205,8 @@ static const struct spinand_info gigadevice_spinand_table[] = { 0, SPINAND_ECCINFO(&gd5fxgq4xa_ooblayout, gd5fxgq4xa_ecc_get_status)), - SPINAND_INFO("GD5F2GQ4xA", 0xF2, + SPINAND_INFO("GD5F2GQ4xA", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0xf2), NAND_MEMORG(1, 2048, 64, 64, 2048, 40, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -213,7 +215,8 @@ static const struct spinand_info gigadevice_spinand_table[] = { 0, SPINAND_ECCINFO(&gd5fxgq4xa_ooblayout, gd5fxgq4xa_ecc_get_status)), - SPINAND_INFO("GD5F4GQ4xA", 0xF4, + SPINAND_INFO("GD5F4GQ4xA", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0xf4), NAND_MEMORG(1, 2048, 64, 64, 4096, 80, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -222,7 +225,8 @@ static const struct spinand_info gigadevice_spinand_table[] = { 0, SPINAND_ECCINFO(&gd5fxgq4xa_ooblayout, gd5fxgq4xa_ecc_get_status)), - SPINAND_INFO("GD5F1GQ4UExxG", 0xd1, + SPINAND_INFO("GD5F1GQ4UExxG", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0xd1), NAND_MEMORG(1, 2048, 128, 64, 1024, 20, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -231,7 +235,8 @@ static const struct spinand_info gigadevice_spinand_table[] = { 0, SPINAND_ECCINFO(&gd5fxgq4_variant2_ooblayout, gd5fxgq4uexxg_ecc_get_status)), - SPINAND_INFO("GD5F1GQ4UFxxG", 0xb148, + SPINAND_INFO("GD5F1GQ4UFxxG", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE, 0xb1, 0x48), NAND_MEMORG(1, 2048, 128, 64, 1024, 20, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants_f, @@ -242,39 +247,13 @@ static const struct spinand_info gigadevice_spinand_table[] = { gd5fxgq4ufxxg_ecc_get_status)), }; -static int gigadevice_spinand_detect(struct spinand_device *spinand) -{ - u8 *id = spinand->id.data; - u16 did; - int ret; - - /* - * Earlier GDF5-series devices (A,E) return [0][MID][DID] - * Later (F) devices return [MID][DID1][DID2] - */ - - if (id[0] == SPINAND_MFR_GIGADEVICE) - did = (id[1] << 8) + id[2]; - else if (id[0] == 0 && id[1] == SPINAND_MFR_GIGADEVICE) - did = id[2]; - else - return 0; - - ret = spinand_match_and_init(spinand, gigadevice_spinand_table, - ARRAY_SIZE(gigadevice_spinand_table), - did); - if (ret) - return ret; - - return 1; -} - static const struct spinand_manufacturer_ops gigadevice_spinand_manuf_ops = { - .detect = gigadevice_spinand_detect, }; const struct spinand_manufacturer gigadevice_spinand_manufacturer = { .id = SPINAND_MFR_GIGADEVICE, .name = "GigaDevice", + .chips = gigadevice_spinand_table, + .nchips = ARRAY_SIZE(gigadevice_spinand_table), .ops = &gigadevice_spinand_manuf_ops, }; diff --git a/drivers/mtd/nand/spi/macronix.c b/drivers/mtd/nand/spi/macronix.c index 21def3f8fb36..0f900f3aa21a 100644 --- a/drivers/mtd/nand/spi/macronix.c +++ b/drivers/mtd/nand/spi/macronix.c @@ -99,7 +99,8 @@ static int mx35lf1ge4ab_ecc_get_status(struct spinand_device *spinand, } static const struct spinand_info macronix_spinand_table[] = { - SPINAND_INFO("MX35LF1GE4AB", 0x12, + SPINAND_INFO("MX35LF1GE4AB", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x12), NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1), NAND_ECCREQ(4, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -108,7 +109,8 @@ static const struct spinand_info macronix_spinand_table[] = { SPINAND_HAS_QE_BIT, SPINAND_ECCINFO(&mx35lfxge4ab_ooblayout, mx35lf1ge4ab_ecc_get_status)), - SPINAND_INFO("MX35LF2GE4AB", 0x22, + SPINAND_INFO("MX35LF2GE4AB", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x22), NAND_MEMORG(1, 2048, 64, 64, 2048, 40, 2, 1, 1), NAND_ECCREQ(4, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -118,33 +120,13 @@ static const struct spinand_info macronix_spinand_table[] = { SPINAND_ECCINFO(&mx35lfxge4ab_ooblayout, NULL)), }; -static int macronix_spinand_detect(struct spinand_device *spinand) -{ - u8 *id = spinand->id.data; - int ret; - - /* - * Macronix SPI NAND read ID needs a dummy byte, so the first byte in - * raw_id is garbage. - */ - if (id[1] != SPINAND_MFR_MACRONIX) - return 0; - - ret = spinand_match_and_init(spinand, macronix_spinand_table, - ARRAY_SIZE(macronix_spinand_table), - id[2]); - if (ret) - return ret; - - return 1; -} - static const struct spinand_manufacturer_ops macronix_spinand_manuf_ops = { - .detect = macronix_spinand_detect, }; const struct spinand_manufacturer macronix_spinand_manufacturer = { .id = SPINAND_MFR_MACRONIX, .name = "Macronix", + .chips = macronix_spinand_table, + .nchips = ARRAY_SIZE(macronix_spinand_table), .ops = ¯onix_spinand_manuf_ops, }; diff --git a/drivers/mtd/nand/spi/micron.c b/drivers/mtd/nand/spi/micron.c index 7d7b1f7fcf71..5d370cfcdaaa 100644 --- a/drivers/mtd/nand/spi/micron.c +++ b/drivers/mtd/nand/spi/micron.c @@ -18,6 +18,16 @@ #define MICRON_STATUS_ECC_4TO6_BITFLIPS (3 << 4) #define MICRON_STATUS_ECC_7TO8_BITFLIPS (5 << 4) +#define MICRON_CFG_CR BIT(0) + +/* + * As per datasheet, die selection is done by the 6th bit of Die + * Select Register (Address 0xD0). + */ +#define MICRON_DIE_SELECT_REG 0xD0 + +#define MICRON_SELECT_DIE(x) ((x) << 6) + static SPINAND_OP_VARIANTS(read_cache_variants, SPINAND_PAGE_READ_FROM_CACHE_QUADIO_OP(0, 2, NULL, 0), SPINAND_PAGE_READ_FROM_CACHE_X4_OP(0, 1, NULL, 0), @@ -34,38 +44,52 @@ static SPINAND_OP_VARIANTS(update_cache_variants, SPINAND_PROG_LOAD_X4(false, 0, NULL, 0), SPINAND_PROG_LOAD(false, 0, NULL, 0)); -static int mt29f2g01abagd_ooblayout_ecc(struct mtd_info *mtd, int section, - struct mtd_oob_region *region) +static int micron_8_ooblayout_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) { if (section) return -ERANGE; - region->offset = 64; - region->length = 64; + region->offset = mtd->oobsize / 2; + region->length = mtd->oobsize / 2; return 0; } -static int mt29f2g01abagd_ooblayout_free(struct mtd_info *mtd, int section, - struct mtd_oob_region *region) +static int micron_8_ooblayout_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) { if (section) return -ERANGE; /* Reserve 2 bytes for the BBM. */ region->offset = 2; - region->length = 62; + region->length = (mtd->oobsize / 2) - 2; return 0; } -static const struct mtd_ooblayout_ops mt29f2g01abagd_ooblayout = { - .ecc = mt29f2g01abagd_ooblayout_ecc, - .free = mt29f2g01abagd_ooblayout_free, +static const struct mtd_ooblayout_ops micron_8_ooblayout = { + .ecc = micron_8_ooblayout_ecc, + .free = micron_8_ooblayout_free, }; -static int mt29f2g01abagd_ecc_get_status(struct spinand_device *spinand, - u8 status) +static int micron_select_target(struct spinand_device *spinand, + unsigned int target) +{ + struct spi_mem_op op = SPINAND_SET_FEATURE_OP(MICRON_DIE_SELECT_REG, + spinand->scratchbuf); + + if (target > 1) + return -EINVAL; + + *spinand->scratchbuf = MICRON_SELECT_DIE(target); + + return spi_mem_exec_op(spinand->spimem, &op); +} + +static int micron_8_ecc_get_status(struct spinand_device *spinand, + u8 status) { switch (status & MICRON_STATUS_ECC_MASK) { case STATUS_ECC_NO_BITFLIPS: @@ -91,43 +115,131 @@ static int mt29f2g01abagd_ecc_get_status(struct spinand_device *spinand, } static const struct spinand_info micron_spinand_table[] = { - SPINAND_INFO("MT29F2G01ABAGD", 0x24, + /* M79A 2Gb 3.3V */ + SPINAND_INFO("MT29F2G01ABAGD", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x24), NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 2, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), 0, - SPINAND_ECCINFO(&mt29f2g01abagd_ooblayout, - mt29f2g01abagd_ecc_get_status)), + SPINAND_ECCINFO(µn_8_ooblayout, + micron_8_ecc_get_status)), + /* M79A 2Gb 1.8V */ + SPINAND_INFO("MT29F2G01ABBGD", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x25), + NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 2, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(µn_8_ooblayout, + micron_8_ecc_get_status)), + /* M78A 1Gb 3.3V */ + SPINAND_INFO("MT29F1G01ABAFD", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x14), + NAND_MEMORG(1, 2048, 128, 64, 1024, 20, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(µn_8_ooblayout, + micron_8_ecc_get_status)), + /* M78A 1Gb 1.8V */ + SPINAND_INFO("MT29F1G01ABAFD", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x15), + NAND_MEMORG(1, 2048, 128, 64, 1024, 20, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(µn_8_ooblayout, + micron_8_ecc_get_status)), + /* M79A 4Gb 3.3V */ + SPINAND_INFO("MT29F4G01ADAGD", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x36), + NAND_MEMORG(1, 2048, 128, 64, 2048, 80, 2, 1, 2), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(µn_8_ooblayout, + micron_8_ecc_get_status), + SPINAND_SELECT_TARGET(micron_select_target)), + /* M70A 4Gb 3.3V */ + SPINAND_INFO("MT29F4G01ABAFD", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x34), + NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + SPINAND_HAS_CR_FEAT_BIT, + SPINAND_ECCINFO(µn_8_ooblayout, + micron_8_ecc_get_status)), + /* M70A 4Gb 1.8V */ + SPINAND_INFO("MT29F4G01ABBFD", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x35), + NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + SPINAND_HAS_CR_FEAT_BIT, + SPINAND_ECCINFO(µn_8_ooblayout, + micron_8_ecc_get_status)), + /* M70A 8Gb 3.3V */ + SPINAND_INFO("MT29F8G01ADAFD", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x46), + NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 2), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + SPINAND_HAS_CR_FEAT_BIT, + SPINAND_ECCINFO(µn_8_ooblayout, + micron_8_ecc_get_status), + SPINAND_SELECT_TARGET(micron_select_target)), + /* M70A 8Gb 1.8V */ + SPINAND_INFO("MT29F8G01ADBFD", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x47), + NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 2), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + SPINAND_HAS_CR_FEAT_BIT, + SPINAND_ECCINFO(µn_8_ooblayout, + micron_8_ecc_get_status), + SPINAND_SELECT_TARGET(micron_select_target)), }; -static int micron_spinand_detect(struct spinand_device *spinand) +static int micron_spinand_init(struct spinand_device *spinand) { - u8 *id = spinand->id.data; - int ret; - /* - * Micron SPI NAND read ID need a dummy byte, - * so the first byte in raw_id is dummy. + * M70A device series enable Continuous Read feature at Power-up, + * which is not supported. Disable this bit to avoid any possible + * failure. */ - if (id[1] != SPINAND_MFR_MICRON) - return 0; + if (spinand->flags & SPINAND_HAS_CR_FEAT_BIT) + return spinand_upd_cfg(spinand, MICRON_CFG_CR, 0); - ret = spinand_match_and_init(spinand, micron_spinand_table, - ARRAY_SIZE(micron_spinand_table), id[2]); - if (ret) - return ret; - - return 1; + return 0; } static const struct spinand_manufacturer_ops micron_spinand_manuf_ops = { - .detect = micron_spinand_detect, + .init = micron_spinand_init, }; const struct spinand_manufacturer micron_spinand_manufacturer = { .id = SPINAND_MFR_MICRON, .name = "Micron", + .chips = micron_spinand_table, + .nchips = ARRAY_SIZE(micron_spinand_table), .ops = µn_spinand_manuf_ops, }; diff --git a/drivers/mtd/nand/spi/paragon.c b/drivers/mtd/nand/spi/paragon.c index 52307681cbd0..519ade513c1f 100644 --- a/drivers/mtd/nand/spi/paragon.c +++ b/drivers/mtd/nand/spi/paragon.c @@ -97,7 +97,8 @@ static const struct mtd_ooblayout_ops pn26g0xa_ooblayout = { static const struct spinand_info paragon_spinand_table[] = { - SPINAND_INFO("PN26G01A", 0xe1, + SPINAND_INFO("PN26G01A", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xe1), NAND_MEMORG(1, 2048, 128, 64, 1024, 21, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -106,7 +107,8 @@ static const struct spinand_info paragon_spinand_table[] = { 0, SPINAND_ECCINFO(&pn26g0xa_ooblayout, pn26g0xa_ecc_get_status)), - SPINAND_INFO("PN26G02A", 0xe2, + SPINAND_INFO("PN26G02A", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xe2), NAND_MEMORG(1, 2048, 128, 64, 2048, 41, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -117,31 +119,13 @@ static const struct spinand_info paragon_spinand_table[] = { pn26g0xa_ecc_get_status)), }; -static int paragon_spinand_detect(struct spinand_device *spinand) -{ - u8 *id = spinand->id.data; - int ret; - - /* Read ID returns [0][MID][DID] */ - - if (id[1] != SPINAND_MFR_PARAGON) - return 0; - - ret = spinand_match_and_init(spinand, paragon_spinand_table, - ARRAY_SIZE(paragon_spinand_table), - id[2]); - if (ret) - return ret; - - return 1; -} - static const struct spinand_manufacturer_ops paragon_spinand_manuf_ops = { - .detect = paragon_spinand_detect, }; const struct spinand_manufacturer paragon_spinand_manufacturer = { .id = SPINAND_MFR_PARAGON, .name = "Paragon", + .chips = paragon_spinand_table, + .nchips = ARRAY_SIZE(paragon_spinand_table), .ops = ¶gon_spinand_manuf_ops, }; diff --git a/drivers/mtd/nand/spi/toshiba.c b/drivers/mtd/nand/spi/toshiba.c index 0db5ee4e82af..bc801d83343e 100644 --- a/drivers/mtd/nand/spi/toshiba.c +++ b/drivers/mtd/nand/spi/toshiba.c @@ -10,6 +10,7 @@ #include #include +/* Kioxia is new name of Toshiba memory. */ #define SPINAND_MFR_TOSHIBA 0x98 #define TOSH_STATUS_ECC_HAS_BITFLIPS_T (3 << 4) @@ -19,14 +20,26 @@ static SPINAND_OP_VARIANTS(read_cache_variants, SPINAND_PAGE_READ_FROM_CACHE_OP(true, 0, 1, NULL, 0), SPINAND_PAGE_READ_FROM_CACHE_OP(false, 0, 1, NULL, 0)); +static SPINAND_OP_VARIANTS(write_cache_x4_variants, + SPINAND_PROG_LOAD_X4(true, 0, NULL, 0), + SPINAND_PROG_LOAD(true, 0, NULL, 0)); + +static SPINAND_OP_VARIANTS(update_cache_x4_variants, + SPINAND_PROG_LOAD_X4(false, 0, NULL, 0), + SPINAND_PROG_LOAD(false, 0, NULL, 0)); + +/** + * Backward compatibility for 1st generation Serial NAND devices + * which don't support Quad Program Load operation. + */ static SPINAND_OP_VARIANTS(write_cache_variants, SPINAND_PROG_LOAD(true, 0, NULL, 0)); static SPINAND_OP_VARIANTS(update_cache_variants, SPINAND_PROG_LOAD(false, 0, NULL, 0)); -static int tc58cxgxsx_ooblayout_ecc(struct mtd_info *mtd, int section, - struct mtd_oob_region *region) +static int tx58cxgxsxraix_ooblayout_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) { if (section > 0) return -ERANGE; @@ -37,8 +50,8 @@ static int tc58cxgxsx_ooblayout_ecc(struct mtd_info *mtd, int section, return 0; } -static int tc58cxgxsx_ooblayout_free(struct mtd_info *mtd, int section, - struct mtd_oob_region *region) +static int tx58cxgxsxraix_ooblayout_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) { if (section > 0) return -ERANGE; @@ -50,13 +63,13 @@ static int tc58cxgxsx_ooblayout_free(struct mtd_info *mtd, int section, return 0; } -static const struct mtd_ooblayout_ops tc58cxgxsx_ooblayout = { - .ecc = tc58cxgxsx_ooblayout_ecc, - .free = tc58cxgxsx_ooblayout_free, +static const struct mtd_ooblayout_ops tx58cxgxsxraix_ooblayout = { + .ecc = tx58cxgxsxraix_ooblayout_ecc, + .free = tx58cxgxsxraix_ooblayout_free, }; -static int tc58cxgxsx_ecc_get_status(struct spinand_device *spinand, - u8 status) +static int tx58cxgxsxraix_ecc_get_status(struct spinand_device *spinand, + u8 status) { struct nand_device *nand = spinand_to_nand(spinand); u8 mbf = 0; @@ -94,105 +107,174 @@ static int tc58cxgxsx_ecc_get_status(struct spinand_device *spinand, } static const struct spinand_info toshiba_spinand_table[] = { - /* 3.3V 1Gb */ - SPINAND_INFO("TC58CVG0S3", 0xC2, + /* 3.3V 1Gb (1st generation) */ + SPINAND_INFO("TC58CVG0S3HRAIG", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xC2), NAND_MEMORG(1, 2048, 128, 64, 1024, 20, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), 0, - SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, - tc58cxgxsx_ecc_get_status)), - /* 3.3V 2Gb */ - SPINAND_INFO("TC58CVG1S3", 0xCB, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 3.3V 2Gb (1st generation) */ + SPINAND_INFO("TC58CVG1S3HRAIG", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xCB), NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), 0, - SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, - tc58cxgxsx_ecc_get_status)), - /* 3.3V 4Gb */ - SPINAND_INFO("TC58CVG2S0", 0xCD, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 3.3V 4Gb (1st generation) */ + SPINAND_INFO("TC58CVG2S0HRAIG", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xCD), NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), 0, - SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, - tc58cxgxsx_ecc_get_status)), - /* 3.3V 4Gb */ - SPINAND_INFO("TC58CVG2S0", 0xED, - NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1), - NAND_ECCREQ(8, 512), - SPINAND_INFO_OP_VARIANTS(&read_cache_variants, - &write_cache_variants, - &update_cache_variants), - 0, - SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, - tc58cxgxsx_ecc_get_status)), - /* 1.8V 1Gb */ - SPINAND_INFO("TC58CYG0S3", 0xB2, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 1.8V 1Gb (1st generation) */ + SPINAND_INFO("TC58CYG0S3HRAIG", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xB2), NAND_MEMORG(1, 2048, 128, 64, 1024, 20, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), 0, - SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, - tc58cxgxsx_ecc_get_status)), - /* 1.8V 2Gb */ - SPINAND_INFO("TC58CYG1S3", 0xBB, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 1.8V 2Gb (1st generation) */ + SPINAND_INFO("TC58CYG1S3HRAIG", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xBB), NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), 0, - SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, - tc58cxgxsx_ecc_get_status)), - /* 1.8V 4Gb */ - SPINAND_INFO("TC58CYG2S0", 0xBD, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 1.8V 4Gb (1st generation) */ + SPINAND_INFO("TC58CYG2S0HRAIG", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xBD), NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), 0, - SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, - tc58cxgxsx_ecc_get_status)), -}; - -static int toshiba_spinand_detect(struct spinand_device *spinand) -{ - u8 *id = spinand->id.data; - int ret; + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), /* - * Toshiba SPI NAND read ID needs a dummy byte, - * so the first byte in id is garbage. + * 2nd generation serial nand has HOLD_D which is equivalent to + * QE_BIT. */ - if (id[1] != SPINAND_MFR_TOSHIBA) - return 0; - - ret = spinand_match_and_init(spinand, toshiba_spinand_table, - ARRAY_SIZE(toshiba_spinand_table), - id[2]); - if (ret) - return ret; - - return 1; -} + /* 3.3V 1Gb (2nd generation) */ + SPINAND_INFO("TC58CVG0S3HRAIJ", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xE2), + NAND_MEMORG(1, 2048, 128, 64, 1024, 20, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_x4_variants, + &update_cache_x4_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 3.3V 2Gb (2nd generation) */ + SPINAND_INFO("TC58CVG1S3HRAIJ", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xEB), + NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_x4_variants, + &update_cache_x4_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 3.3V 4Gb (2nd generation) */ + SPINAND_INFO("TC58CVG2S0HRAIJ", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xED), + NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_x4_variants, + &update_cache_x4_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 3.3V 8Gb (2nd generation) */ + SPINAND_INFO("TH58CVG3S0HRAIJ", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xE4), + NAND_MEMORG(1, 4096, 256, 64, 4096, 80, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_x4_variants, + &update_cache_x4_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 1.8V 1Gb (2nd generation) */ + SPINAND_INFO("TC58CYG0S3HRAIJ", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xD2), + NAND_MEMORG(1, 2048, 128, 64, 1024, 20, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_x4_variants, + &update_cache_x4_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 1.8V 2Gb (2nd generation) */ + SPINAND_INFO("TC58CYG1S3HRAIJ", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xDB), + NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_x4_variants, + &update_cache_x4_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 1.8V 4Gb (2nd generation) */ + SPINAND_INFO("TC58CYG2S0HRAIJ", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xDD), + NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_x4_variants, + &update_cache_x4_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), + /* 1.8V 8Gb (2nd generation) */ + SPINAND_INFO("TH58CYG3S0HRAIJ", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xD4), + NAND_MEMORG(1, 4096, 256, 64, 4096, 80, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_x4_variants, + &update_cache_x4_variants), + SPINAND_HAS_QE_BIT, + SPINAND_ECCINFO(&tx58cxgxsxraix_ooblayout, + tx58cxgxsxraix_ecc_get_status)), +}; static const struct spinand_manufacturer_ops toshiba_spinand_manuf_ops = { - .detect = toshiba_spinand_detect, }; const struct spinand_manufacturer toshiba_spinand_manufacturer = { .id = SPINAND_MFR_TOSHIBA, .name = "Toshiba", + .chips = toshiba_spinand_table, + .nchips = ARRAY_SIZE(toshiba_spinand_table), .ops = &toshiba_spinand_manuf_ops, }; diff --git a/drivers/mtd/nand/spi/winbond.c b/drivers/mtd/nand/spi/winbond.c index a6c17e0cace8..76684428354e 100644 --- a/drivers/mtd/nand/spi/winbond.c +++ b/drivers/mtd/nand/spi/winbond.c @@ -75,7 +75,8 @@ static int w25m02gv_select_target(struct spinand_device *spinand, } static const struct spinand_info winbond_spinand_table[] = { - SPINAND_INFO("W25M02GV", 0xAB, + SPINAND_INFO("W25M02GV", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xab), NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 2), NAND_ECCREQ(1, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -84,7 +85,8 @@ static const struct spinand_info winbond_spinand_table[] = { 0, SPINAND_ECCINFO(&w25m02gv_ooblayout, NULL), SPINAND_SELECT_TARGET(w25m02gv_select_target)), - SPINAND_INFO("W25N01GV", 0xAA, + SPINAND_INFO("W25N01GV", + SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xaa), NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1), NAND_ECCREQ(1, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, @@ -94,31 +96,6 @@ static const struct spinand_info winbond_spinand_table[] = { SPINAND_ECCINFO(&w25m02gv_ooblayout, NULL)), }; -/** - * winbond_spinand_detect - initialize device related part in spinand_device - * struct if it is a Winbond device. - * @spinand: SPI NAND device structure - */ -static int winbond_spinand_detect(struct spinand_device *spinand) -{ - u8 *id = spinand->id.data; - int ret; - - /* - * Winbond SPI NAND read ID need a dummy byte, - * so the first byte in raw_id is dummy. - */ - if (id[1] != SPINAND_MFR_WINBOND) - return 0; - - ret = spinand_match_and_init(spinand, winbond_spinand_table, - ARRAY_SIZE(winbond_spinand_table), id[2]); - if (ret) - return ret; - - return 1; -} - static int winbond_spinand_init(struct spinand_device *spinand) { struct nand_device *nand = spinand_to_nand(spinand); @@ -138,12 +115,13 @@ static int winbond_spinand_init(struct spinand_device *spinand) } static const struct spinand_manufacturer_ops winbond_spinand_manuf_ops = { - .detect = winbond_spinand_detect, .init = winbond_spinand_init, }; const struct spinand_manufacturer winbond_spinand_manufacturer = { .id = SPINAND_MFR_WINBOND, .name = "Winbond", + .chips = winbond_spinand_table, + .nchips = ARRAY_SIZE(winbond_spinand_table), .ops = &winbond_spinand_manuf_ops, }; diff --git a/drivers/mtd/spi-nor/Kconfig b/drivers/mtd/spi-nor/Kconfig index 267b9000782e..6e816eafb312 100644 --- a/drivers/mtd/spi-nor/Kconfig +++ b/drivers/mtd/spi-nor/Kconfig @@ -24,79 +24,6 @@ config MTD_SPI_NOR_USE_4K_SECTORS Please note that some tools/drivers/filesystems may not work with 4096 B erase size (e.g. UBIFS requires 15 KiB as a minimum). -config SPI_ASPEED_SMC - tristate "Aspeed flash controllers in SPI mode" - depends on ARCH_ASPEED || COMPILE_TEST - depends on HAS_IOMEM && OF - help - This enables support for the Firmware Memory controller (FMC) - in the Aspeed AST2500/AST2400 SoCs when attached to SPI NOR chips, - and support for the SPI flash memory controller (SPI) for - the host firmware. The implementation only supports SPI NOR. - -config SPI_CADENCE_QUADSPI - tristate "Cadence Quad SPI controller" - depends on OF && (ARM || ARM64 || COMPILE_TEST) - help - Enable support for the Cadence Quad SPI Flash controller. - - Cadence QSPI is a specialized controller for connecting an SPI - Flash over 1/2/4-bit wide bus. Enable this option if you have a - device with a Cadence QSPI controller and want to access the - Flash as an MTD device. - -config SPI_HISI_SFC - tristate "Hisilicon FMC SPI-NOR Flash Controller(SFC)" - depends on ARCH_HISI || COMPILE_TEST - depends on HAS_IOMEM - help - This enables support for HiSilicon FMC SPI-NOR flash controller. - -config SPI_NXP_SPIFI - tristate "NXP SPI Flash Interface (SPIFI)" - depends on OF && (ARCH_LPC18XX || COMPILE_TEST) - depends on HAS_IOMEM - help - Enable support for the NXP LPC SPI Flash Interface controller. - - SPIFI is a specialized controller for connecting serial SPI - Flash. Enable this option if you have a device with a SPIFI - controller and want to access the Flash as a mtd device. - -config SPI_INTEL_SPI - tristate - -config SPI_INTEL_SPI_PCI - tristate "Intel PCH/PCU SPI flash PCI driver (DANGEROUS)" - depends on X86 && PCI - select SPI_INTEL_SPI - help - This enables PCI support for the Intel PCH/PCU SPI controller in - master mode. This controller is present in modern Intel hardware - and is used to hold BIOS and other persistent settings. Using - this driver it is possible to upgrade BIOS directly from Linux. - - Say N here unless you know what you are doing. Overwriting the - SPI flash may render the system unbootable. - - To compile this driver as a module, choose M here: the module - will be called intel-spi-pci. - -config SPI_INTEL_SPI_PLATFORM - tristate "Intel PCH/PCU SPI flash platform driver (DANGEROUS)" - depends on X86 - select SPI_INTEL_SPI - help - This enables platform support for the Intel PCH/PCU SPI - controller in master mode. This controller is present in modern - Intel hardware and is used to hold BIOS and other persistent - settings. Using this driver it is possible to upgrade BIOS - directly from Linux. - - Say N here unless you know what you are doing. Overwriting the - SPI flash may render the system unbootable. - - To compile this driver as a module, choose M here: the module - will be called intel-spi-platform. +source "drivers/mtd/spi-nor/controllers/Kconfig" endif # MTD_SPI_NOR diff --git a/drivers/mtd/spi-nor/Makefile b/drivers/mtd/spi-nor/Makefile index 738dfd74cf76..7ddb742de1fe 100644 --- a/drivers/mtd/spi-nor/Makefile +++ b/drivers/mtd/spi-nor/Makefile @@ -1,9 +1,20 @@ # SPDX-License-Identifier: GPL-2.0 + +spi-nor-objs := core.o sfdp.o +spi-nor-objs += atmel.o +spi-nor-objs += catalyst.o +spi-nor-objs += eon.o +spi-nor-objs += esmt.o +spi-nor-objs += everspin.o +spi-nor-objs += fujitsu.o +spi-nor-objs += gigadevice.o +spi-nor-objs += intel.o +spi-nor-objs += issi.o +spi-nor-objs += macronix.o +spi-nor-objs += micron-st.o +spi-nor-objs += spansion.o +spi-nor-objs += sst.o +spi-nor-objs += winbond.o +spi-nor-objs += xilinx.o +spi-nor-objs += xmc.o obj-$(CONFIG_MTD_SPI_NOR) += spi-nor.o -obj-$(CONFIG_SPI_ASPEED_SMC) += aspeed-smc.o -obj-$(CONFIG_SPI_CADENCE_QUADSPI) += cadence-quadspi.o -obj-$(CONFIG_SPI_HISI_SFC) += hisi-sfc.o -obj-$(CONFIG_SPI_NXP_SPIFI) += nxp-spifi.o -obj-$(CONFIG_SPI_INTEL_SPI) += intel-spi.o -obj-$(CONFIG_SPI_INTEL_SPI_PCI) += intel-spi-pci.o -obj-$(CONFIG_SPI_INTEL_SPI_PLATFORM) += intel-spi-platform.o diff --git a/drivers/mtd/spi-nor/atmel.c b/drivers/mtd/spi-nor/atmel.c new file mode 100644 index 000000000000..3f5f21a473a6 --- /dev/null +++ b/drivers/mtd/spi-nor/atmel.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info atmel_parts[] = { + /* Atmel -- some are (confusingly) marketed as "DataFlash" */ + { "at25fs010", INFO(0x1f6601, 0, 32 * 1024, 4, SECT_4K) }, + { "at25fs040", INFO(0x1f6604, 0, 64 * 1024, 8, SECT_4K) }, + + { "at25df041a", INFO(0x1f4401, 0, 64 * 1024, 8, SECT_4K) }, + { "at25df321", INFO(0x1f4700, 0, 64 * 1024, 64, SECT_4K) }, + { "at25df321a", INFO(0x1f4701, 0, 64 * 1024, 64, SECT_4K) }, + { "at25df641", INFO(0x1f4800, 0, 64 * 1024, 128, SECT_4K) }, + + { "at25sl321", INFO(0x1f4216, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + + { "at26f004", INFO(0x1f0400, 0, 64 * 1024, 8, SECT_4K) }, + { "at26df081a", INFO(0x1f4501, 0, 64 * 1024, 16, SECT_4K) }, + { "at26df161a", INFO(0x1f4601, 0, 64 * 1024, 32, SECT_4K) }, + { "at26df321", INFO(0x1f4700, 0, 64 * 1024, 64, SECT_4K) }, + + { "at45db081d", INFO(0x1f2500, 0, 64 * 1024, 16, SECT_4K) }, +}; + +static void atmel_default_init(struct spi_nor *nor) +{ + nor->flags |= SNOR_F_HAS_LOCK; +} + +static const struct spi_nor_fixups atmel_fixups = { + .default_init = atmel_default_init, +}; + +const struct spi_nor_manufacturer spi_nor_atmel = { + .name = "atmel", + .parts = atmel_parts, + .nparts = ARRAY_SIZE(atmel_parts), + .fixups = &atmel_fixups, +}; diff --git a/drivers/mtd/spi-nor/catalyst.c b/drivers/mtd/spi-nor/catalyst.c new file mode 100644 index 000000000000..011b83e99e95 --- /dev/null +++ b/drivers/mtd/spi-nor/catalyst.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info catalyst_parts[] = { + /* Catalyst / On Semiconductor -- non-JEDEC */ + { "cat25c11", CAT25_INFO(16, 8, 16, 1, + SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, + { "cat25c03", CAT25_INFO(32, 8, 16, 2, + SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, + { "cat25c09", CAT25_INFO(128, 8, 32, 2, + SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, + { "cat25c17", CAT25_INFO(256, 8, 32, 2, + SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, + { "cat25128", CAT25_INFO(2048, 8, 64, 2, + SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, +}; + +const struct spi_nor_manufacturer spi_nor_catalyst = { + .name = "catalyst", + .parts = catalyst_parts, + .nparts = ARRAY_SIZE(catalyst_parts), +}; diff --git a/drivers/mtd/spi-nor/controllers/Kconfig b/drivers/mtd/spi-nor/controllers/Kconfig new file mode 100644 index 000000000000..10b86660b821 --- /dev/null +++ b/drivers/mtd/spi-nor/controllers/Kconfig @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: GPL-2.0-only +config SPI_ASPEED_SMC + tristate "Aspeed flash controllers in SPI mode" + depends on ARCH_ASPEED || COMPILE_TEST + depends on HAS_IOMEM && OF + help + This enables support for the Firmware Memory controller (FMC) + in the Aspeed AST2500/AST2400 SoCs when attached to SPI NOR chips, + and support for the SPI flash memory controller (SPI) for + the host firmware. The implementation only supports SPI NOR. + +config SPI_CADENCE_QUADSPI + tristate "Cadence Quad SPI controller" + depends on OF && (ARM || ARM64 || COMPILE_TEST) + help + Enable support for the Cadence Quad SPI Flash controller. + + Cadence QSPI is a specialized controller for connecting an SPI + Flash over 1/2/4-bit wide bus. Enable this option if you have a + device with a Cadence QSPI controller and want to access the + Flash as an MTD device. + +config SPI_HISI_SFC + tristate "Hisilicon FMC SPI-NOR Flash Controller(SFC)" + depends on ARCH_HISI || COMPILE_TEST + depends on HAS_IOMEM + help + This enables support for HiSilicon FMC SPI-NOR flash controller. + +config SPI_NXP_SPIFI + tristate "NXP SPI Flash Interface (SPIFI)" + depends on OF && (ARCH_LPC18XX || COMPILE_TEST) + depends on HAS_IOMEM + help + Enable support for the NXP LPC SPI Flash Interface controller. + + SPIFI is a specialized controller for connecting serial SPI + Flash. Enable this option if you have a device with a SPIFI + controller and want to access the Flash as a mtd device. + +config SPI_INTEL_SPI + tristate + +config SPI_INTEL_SPI_PCI + tristate "Intel PCH/PCU SPI flash PCI driver (DANGEROUS)" + depends on X86 && PCI + select SPI_INTEL_SPI + help + This enables PCI support for the Intel PCH/PCU SPI controller in + master mode. This controller is present in modern Intel hardware + and is used to hold BIOS and other persistent settings. Using + this driver it is possible to upgrade BIOS directly from Linux. + + Say N here unless you know what you are doing. Overwriting the + SPI flash may render the system unbootable. + + To compile this driver as a module, choose M here: the module + will be called intel-spi-pci. + +config SPI_INTEL_SPI_PLATFORM + tristate "Intel PCH/PCU SPI flash platform driver (DANGEROUS)" + depends on X86 + select SPI_INTEL_SPI + help + This enables platform support for the Intel PCH/PCU SPI + controller in master mode. This controller is present in modern + Intel hardware and is used to hold BIOS and other persistent + settings. Using this driver it is possible to upgrade BIOS + directly from Linux. + + Say N here unless you know what you are doing. Overwriting the + SPI flash may render the system unbootable. + + To compile this driver as a module, choose M here: the module + will be called intel-spi-platform. diff --git a/drivers/mtd/spi-nor/controllers/Makefile b/drivers/mtd/spi-nor/controllers/Makefile new file mode 100644 index 000000000000..46e6fbe586e3 --- /dev/null +++ b/drivers/mtd/spi-nor/controllers/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_SPI_ASPEED_SMC) += aspeed-smc.o +obj-$(CONFIG_SPI_CADENCE_QUADSPI) += cadence-quadspi.o +obj-$(CONFIG_SPI_HISI_SFC) += hisi-sfc.o +obj-$(CONFIG_SPI_NXP_SPIFI) += nxp-spifi.o +obj-$(CONFIG_SPI_INTEL_SPI) += intel-spi.o +obj-$(CONFIG_SPI_INTEL_SPI_PCI) += intel-spi-pci.o +obj-$(CONFIG_SPI_INTEL_SPI_PLATFORM) += intel-spi-platform.o diff --git a/drivers/mtd/spi-nor/aspeed-smc.c b/drivers/mtd/spi-nor/controllers/aspeed-smc.c similarity index 99% rename from drivers/mtd/spi-nor/aspeed-smc.c rename to drivers/mtd/spi-nor/controllers/aspeed-smc.c index 395127349aa8..ae85e4c0e114 100644 --- a/drivers/mtd/spi-nor/aspeed-smc.c +++ b/drivers/mtd/spi-nor/controllers/aspeed-smc.c @@ -109,7 +109,7 @@ struct aspeed_smc_controller { void __iomem *ahb_base; /* per-chip windows resource */ u32 ahb_window_size; /* full mapping window size */ - struct aspeed_smc_chip *chips[0]; /* pointers to attached chips */ + struct aspeed_smc_chip *chips[]; /* pointers to attached chips */ }; /* @@ -354,7 +354,7 @@ static void aspeed_smc_send_cmd_addr(struct spi_nor *nor, u8 cmd, u32 addr) default: WARN_ONCE(1, "Unexpected address width %u, defaulting to 3\n", nor->addr_width); - /* FALLTHROUGH */ + fallthrough; case 3: cmdaddr = addr & 0xFFFFFF; cmdaddr |= cmd << 24; diff --git a/drivers/mtd/spi-nor/cadence-quadspi.c b/drivers/mtd/spi-nor/controllers/cadence-quadspi.c similarity index 100% rename from drivers/mtd/spi-nor/cadence-quadspi.c rename to drivers/mtd/spi-nor/controllers/cadence-quadspi.c diff --git a/drivers/mtd/spi-nor/hisi-sfc.c b/drivers/mtd/spi-nor/controllers/hisi-sfc.c similarity index 100% rename from drivers/mtd/spi-nor/hisi-sfc.c rename to drivers/mtd/spi-nor/controllers/hisi-sfc.c diff --git a/drivers/mtd/spi-nor/intel-spi-pci.c b/drivers/mtd/spi-nor/controllers/intel-spi-pci.c similarity index 100% rename from drivers/mtd/spi-nor/intel-spi-pci.c rename to drivers/mtd/spi-nor/controllers/intel-spi-pci.c diff --git a/drivers/mtd/spi-nor/intel-spi-platform.c b/drivers/mtd/spi-nor/controllers/intel-spi-platform.c similarity index 100% rename from drivers/mtd/spi-nor/intel-spi-platform.c rename to drivers/mtd/spi-nor/controllers/intel-spi-platform.c diff --git a/drivers/mtd/spi-nor/intel-spi.c b/drivers/mtd/spi-nor/controllers/intel-spi.c similarity index 100% rename from drivers/mtd/spi-nor/intel-spi.c rename to drivers/mtd/spi-nor/controllers/intel-spi.c diff --git a/drivers/mtd/spi-nor/intel-spi.h b/drivers/mtd/spi-nor/controllers/intel-spi.h similarity index 100% rename from drivers/mtd/spi-nor/intel-spi.h rename to drivers/mtd/spi-nor/controllers/intel-spi.h diff --git a/drivers/mtd/spi-nor/nxp-spifi.c b/drivers/mtd/spi-nor/controllers/nxp-spifi.c similarity index 100% rename from drivers/mtd/spi-nor/nxp-spifi.c rename to drivers/mtd/spi-nor/controllers/nxp-spifi.c diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c new file mode 100644 index 000000000000..cc68ea84318e --- /dev/null +++ b/drivers/mtd/spi-nor/core.c @@ -0,0 +1,3466 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Based on m25p80.c, by Mike Lavender (mike@steroidmicros.com), with + * influence from lart.c (Abraham Van Der Merwe) and mtd_dataflash.c + * + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "core.h" + +/* Define max times to check status register before we give up. */ + +/* + * For everything but full-chip erase; probably could be much smaller, but kept + * around for safety for now + */ +#define DEFAULT_READY_WAIT_JIFFIES (40UL * HZ) + +/* + * For full-chip erase, calibrated to a 2MB flash (M25P16); should be scaled up + * for larger flash + */ +#define CHIP_ERASE_2MB_READY_WAIT_JIFFIES (40UL * HZ) + +#define SPI_NOR_MAX_ADDR_WIDTH 4 + +/** + * spi_nor_spimem_bounce() - check if a bounce buffer is needed for the data + * transfer + * @nor: pointer to 'struct spi_nor' + * @op: pointer to 'struct spi_mem_op' template for transfer + * + * If we have to use the bounce buffer, the data field in @op will be updated. + * + * Return: true if the bounce buffer is needed, false if not + */ +static bool spi_nor_spimem_bounce(struct spi_nor *nor, struct spi_mem_op *op) +{ + /* op->data.buf.in occupies the same memory as op->data.buf.out */ + if (object_is_on_stack(op->data.buf.in) || + !virt_addr_valid(op->data.buf.in)) { + if (op->data.nbytes > nor->bouncebuf_size) + op->data.nbytes = nor->bouncebuf_size; + op->data.buf.in = nor->bouncebuf; + return true; + } + + return false; +} + +/** + * spi_nor_spimem_exec_op() - execute a memory operation + * @nor: pointer to 'struct spi_nor' + * @op: pointer to 'struct spi_mem_op' template for transfer + * + * Return: 0 on success, -error otherwise. + */ +static int spi_nor_spimem_exec_op(struct spi_nor *nor, struct spi_mem_op *op) +{ + int error; + + error = spi_mem_adjust_op_size(nor->spimem, op); + if (error) + return error; + + return spi_mem_exec_op(nor->spimem, op); +} + +/** + * spi_nor_spimem_read_data() - read data from flash's memory region via + * spi-mem + * @nor: pointer to 'struct spi_nor' + * @from: offset to read from + * @len: number of bytes to read + * @buf: pointer to dst buffer + * + * Return: number of bytes read successfully, -errno otherwise + */ +static ssize_t spi_nor_spimem_read_data(struct spi_nor *nor, loff_t from, + size_t len, u8 *buf) +{ + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(nor->read_opcode, 1), + SPI_MEM_OP_ADDR(nor->addr_width, from, 1), + SPI_MEM_OP_DUMMY(nor->read_dummy, 1), + SPI_MEM_OP_DATA_IN(len, buf, 1)); + bool usebouncebuf; + ssize_t nbytes; + int error; + + /* get transfer protocols. */ + op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(nor->read_proto); + op.addr.buswidth = spi_nor_get_protocol_addr_nbits(nor->read_proto); + op.dummy.buswidth = op.addr.buswidth; + op.data.buswidth = spi_nor_get_protocol_data_nbits(nor->read_proto); + + /* convert the dummy cycles to the number of bytes */ + op.dummy.nbytes = (nor->read_dummy * op.dummy.buswidth) / 8; + + usebouncebuf = spi_nor_spimem_bounce(nor, &op); + + if (nor->dirmap.rdesc) { + nbytes = spi_mem_dirmap_read(nor->dirmap.rdesc, op.addr.val, + op.data.nbytes, op.data.buf.in); + } else { + error = spi_nor_spimem_exec_op(nor, &op); + if (error) + return error; + nbytes = op.data.nbytes; + } + + if (usebouncebuf && nbytes > 0) + memcpy(buf, op.data.buf.in, nbytes); + + return nbytes; +} + +/** + * spi_nor_read_data() - read data from flash memory + * @nor: pointer to 'struct spi_nor' + * @from: offset to read from + * @len: number of bytes to read + * @buf: pointer to dst buffer + * + * Return: number of bytes read successfully, -errno otherwise + */ +ssize_t spi_nor_read_data(struct spi_nor *nor, loff_t from, size_t len, u8 *buf) +{ + if (nor->spimem) + return spi_nor_spimem_read_data(nor, from, len, buf); + + return nor->controller_ops->read(nor, from, len, buf); +} + +/** + * spi_nor_spimem_write_data() - write data to flash memory via + * spi-mem + * @nor: pointer to 'struct spi_nor' + * @to: offset to write to + * @len: number of bytes to write + * @buf: pointer to src buffer + * + * Return: number of bytes written successfully, -errno otherwise + */ +static ssize_t spi_nor_spimem_write_data(struct spi_nor *nor, loff_t to, + size_t len, const u8 *buf) +{ + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(nor->program_opcode, 1), + SPI_MEM_OP_ADDR(nor->addr_width, to, 1), + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_OUT(len, buf, 1)); + ssize_t nbytes; + int error; + + op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(nor->write_proto); + op.addr.buswidth = spi_nor_get_protocol_addr_nbits(nor->write_proto); + op.data.buswidth = spi_nor_get_protocol_data_nbits(nor->write_proto); + + if (nor->program_opcode == SPINOR_OP_AAI_WP && nor->sst_write_second) + op.addr.nbytes = 0; + + if (spi_nor_spimem_bounce(nor, &op)) + memcpy(nor->bouncebuf, buf, op.data.nbytes); + + if (nor->dirmap.wdesc) { + nbytes = spi_mem_dirmap_write(nor->dirmap.wdesc, op.addr.val, + op.data.nbytes, op.data.buf.out); + } else { + error = spi_nor_spimem_exec_op(nor, &op); + if (error) + return error; + nbytes = op.data.nbytes; + } + + return nbytes; +} + +/** + * spi_nor_write_data() - write data to flash memory + * @nor: pointer to 'struct spi_nor' + * @to: offset to write to + * @len: number of bytes to write + * @buf: pointer to src buffer + * + * Return: number of bytes written successfully, -errno otherwise + */ +ssize_t spi_nor_write_data(struct spi_nor *nor, loff_t to, size_t len, + const u8 *buf) +{ + if (nor->spimem) + return spi_nor_spimem_write_data(nor, to, len, buf); + + return nor->controller_ops->write(nor, to, len, buf); +} + +/** + * spi_nor_write_enable() - Set write enable latch with Write Enable command. + * @nor: pointer to 'struct spi_nor'. + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_write_enable(struct spi_nor *nor) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WREN, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_NO_DATA); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WREN, + NULL, 0); + } + + if (ret) + dev_dbg(nor->dev, "error %d on Write Enable\n", ret); + + return ret; +} + +/** + * spi_nor_write_disable() - Send Write Disable instruction to the chip. + * @nor: pointer to 'struct spi_nor'. + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_write_disable(struct spi_nor *nor) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WRDI, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_NO_DATA); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WRDI, + NULL, 0); + } + + if (ret) + dev_dbg(nor->dev, "error %d on Write Disable\n", ret); + + return ret; +} + +/** + * spi_nor_read_sr() - Read the Status Register. + * @nor: pointer to 'struct spi_nor'. + * @sr: pointer to a DMA-able buffer where the value of the + * Status Register will be written. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_read_sr(struct spi_nor *nor, u8 *sr) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDSR, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_IN(1, sr, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->read_reg(nor, SPINOR_OP_RDSR, + sr, 1); + } + + if (ret) + dev_dbg(nor->dev, "error %d reading SR\n", ret); + + return ret; +} + +/** + * spi_nor_read_fsr() - Read the Flag Status Register. + * @nor: pointer to 'struct spi_nor' + * @fsr: pointer to a DMA-able buffer where the value of the + * Flag Status Register will be written. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_read_fsr(struct spi_nor *nor, u8 *fsr) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDFSR, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_IN(1, fsr, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->read_reg(nor, SPINOR_OP_RDFSR, + fsr, 1); + } + + if (ret) + dev_dbg(nor->dev, "error %d reading FSR\n", ret); + + return ret; +} + +/** + * spi_nor_read_cr() - Read the Configuration Register using the + * SPINOR_OP_RDCR (35h) command. + * @nor: pointer to 'struct spi_nor' + * @cr: pointer to a DMA-able buffer where the value of the + * Configuration Register will be written. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_read_cr(struct spi_nor *nor, u8 *cr) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDCR, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_IN(1, cr, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->read_reg(nor, SPINOR_OP_RDCR, cr, 1); + } + + if (ret) + dev_dbg(nor->dev, "error %d reading CR\n", ret); + + return ret; +} + +/** + * spi_nor_set_4byte_addr_mode() - Enter/Exit 4-byte address mode. + * @nor: pointer to 'struct spi_nor'. + * @enable: true to enter the 4-byte address mode, false to exit the 4-byte + * address mode. + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(enable ? + SPINOR_OP_EN4B : + SPINOR_OP_EX4B, + 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_NO_DATA); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, + enable ? SPINOR_OP_EN4B : + SPINOR_OP_EX4B, + NULL, 0); + } + + if (ret) + dev_dbg(nor->dev, "error %d setting 4-byte mode\n", ret); + + return ret; +} + +/** + * spansion_set_4byte_addr_mode() - Set 4-byte address mode for Spansion + * flashes. + * @nor: pointer to 'struct spi_nor'. + * @enable: true to enter the 4-byte address mode, false to exit the 4-byte + * address mode. + * + * Return: 0 on success, -errno otherwise. + */ +static int spansion_set_4byte_addr_mode(struct spi_nor *nor, bool enable) +{ + int ret; + + nor->bouncebuf[0] = enable << 7; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_BRWR, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_OUT(1, nor->bouncebuf, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, SPINOR_OP_BRWR, + nor->bouncebuf, 1); + } + + if (ret) + dev_dbg(nor->dev, "error %d setting 4-byte mode\n", ret); + + return ret; +} + +/** + * spi_nor_write_ear() - Write Extended Address Register. + * @nor: pointer to 'struct spi_nor'. + * @ear: value to write to the Extended Address Register. + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_write_ear(struct spi_nor *nor, u8 ear) +{ + int ret; + + nor->bouncebuf[0] = ear; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WREAR, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_OUT(1, nor->bouncebuf, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WREAR, + nor->bouncebuf, 1); + } + + if (ret) + dev_dbg(nor->dev, "error %d writing EAR\n", ret); + + return ret; +} + +/** + * spi_nor_xread_sr() - Read the Status Register on S3AN flashes. + * @nor: pointer to 'struct spi_nor'. + * @sr: pointer to a DMA-able buffer where the value of the + * Status Register will be written. + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_xread_sr(struct spi_nor *nor, u8 *sr) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_XRDSR, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_IN(1, sr, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->read_reg(nor, SPINOR_OP_XRDSR, + sr, 1); + } + + if (ret) + dev_dbg(nor->dev, "error %d reading XRDSR\n", ret); + + return ret; +} + +/** + * spi_nor_xsr_ready() - Query the Status Register of the S3AN flash to see if + * the flash is ready for new commands. + * @nor: pointer to 'struct spi_nor'. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_xsr_ready(struct spi_nor *nor) +{ + int ret; + + ret = spi_nor_xread_sr(nor, nor->bouncebuf); + if (ret) + return ret; + + return !!(nor->bouncebuf[0] & XSR_RDY); +} + +/** + * spi_nor_clear_sr() - Clear the Status Register. + * @nor: pointer to 'struct spi_nor'. + */ +static void spi_nor_clear_sr(struct spi_nor *nor) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_CLSR, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_NO_DATA); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, SPINOR_OP_CLSR, + NULL, 0); + } + + if (ret) + dev_dbg(nor->dev, "error %d clearing SR\n", ret); +} + +/** + * spi_nor_sr_ready() - Query the Status Register to see if the flash is ready + * for new commands. + * @nor: pointer to 'struct spi_nor'. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_sr_ready(struct spi_nor *nor) +{ + int ret = spi_nor_read_sr(nor, nor->bouncebuf); + + if (ret) + return ret; + + if (nor->flags & SNOR_F_USE_CLSR && + nor->bouncebuf[0] & (SR_E_ERR | SR_P_ERR)) { + if (nor->bouncebuf[0] & SR_E_ERR) + dev_err(nor->dev, "Erase Error occurred\n"); + else + dev_err(nor->dev, "Programming Error occurred\n"); + + spi_nor_clear_sr(nor); + + /* + * WEL bit remains set to one when an erase or page program + * error occurs. Issue a Write Disable command to protect + * against inadvertent writes that can possibly corrupt the + * contents of the memory. + */ + ret = spi_nor_write_disable(nor); + if (ret) + return ret; + + return -EIO; + } + + return !(nor->bouncebuf[0] & SR_WIP); +} + +/** + * spi_nor_clear_fsr() - Clear the Flag Status Register. + * @nor: pointer to 'struct spi_nor'. + */ +static void spi_nor_clear_fsr(struct spi_nor *nor) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_CLFSR, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_NO_DATA); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, SPINOR_OP_CLFSR, + NULL, 0); + } + + if (ret) + dev_dbg(nor->dev, "error %d clearing FSR\n", ret); +} + +/** + * spi_nor_fsr_ready() - Query the Flag Status Register to see if the flash is + * ready for new commands. + * @nor: pointer to 'struct spi_nor'. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_fsr_ready(struct spi_nor *nor) +{ + int ret = spi_nor_read_fsr(nor, nor->bouncebuf); + + if (ret) + return ret; + + if (nor->bouncebuf[0] & (FSR_E_ERR | FSR_P_ERR)) { + if (nor->bouncebuf[0] & FSR_E_ERR) + dev_err(nor->dev, "Erase operation failed.\n"); + else + dev_err(nor->dev, "Program operation failed.\n"); + + if (nor->bouncebuf[0] & FSR_PT_ERR) + dev_err(nor->dev, + "Attempted to modify a protected sector.\n"); + + spi_nor_clear_fsr(nor); + + /* + * WEL bit remains set to one when an erase or page program + * error occurs. Issue a Write Disable command to protect + * against inadvertent writes that can possibly corrupt the + * contents of the memory. + */ + ret = spi_nor_write_disable(nor); + if (ret) + return ret; + + return -EIO; + } + + return nor->bouncebuf[0] & FSR_READY; +} + +/** + * spi_nor_ready() - Query the flash to see if it is ready for new commands. + * @nor: pointer to 'struct spi_nor'. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_ready(struct spi_nor *nor) +{ + int sr, fsr; + + if (nor->flags & SNOR_F_READY_XSR_RDY) + sr = spi_nor_xsr_ready(nor); + else + sr = spi_nor_sr_ready(nor); + if (sr < 0) + return sr; + fsr = nor->flags & SNOR_F_USE_FSR ? spi_nor_fsr_ready(nor) : 1; + if (fsr < 0) + return fsr; + return sr && fsr; +} + +/** + * spi_nor_wait_till_ready_with_timeout() - Service routine to read the + * Status Register until ready, or timeout occurs. + * @nor: pointer to "struct spi_nor". + * @timeout_jiffies: jiffies to wait until timeout. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_wait_till_ready_with_timeout(struct spi_nor *nor, + unsigned long timeout_jiffies) +{ + unsigned long deadline; + int timeout = 0, ret; + + deadline = jiffies + timeout_jiffies; + + while (!timeout) { + if (time_after_eq(jiffies, deadline)) + timeout = 1; + + ret = spi_nor_ready(nor); + if (ret < 0) + return ret; + if (ret) + return 0; + + cond_resched(); + } + + dev_dbg(nor->dev, "flash operation timed out\n"); + + return -ETIMEDOUT; +} + +/** + * spi_nor_wait_till_ready() - Wait for a predefined amount of time for the + * flash to be ready, or timeout occurs. + * @nor: pointer to "struct spi_nor". + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_wait_till_ready(struct spi_nor *nor) +{ + return spi_nor_wait_till_ready_with_timeout(nor, + DEFAULT_READY_WAIT_JIFFIES); +} + +/** + * spi_nor_write_sr() - Write the Status Register. + * @nor: pointer to 'struct spi_nor'. + * @sr: pointer to DMA-able buffer to write to the Status Register. + * @len: number of bytes to write to the Status Register. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_write_sr(struct spi_nor *nor, const u8 *sr, size_t len) +{ + int ret; + + ret = spi_nor_write_enable(nor); + if (ret) + return ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WRSR, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_OUT(len, sr, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WRSR, + sr, len); + } + + if (ret) { + dev_dbg(nor->dev, "error %d writing SR\n", ret); + return ret; + } + + return spi_nor_wait_till_ready(nor); +} + +/** + * spi_nor_write_sr1_and_check() - Write one byte to the Status Register 1 and + * ensure that the byte written match the received value. + * @nor: pointer to a 'struct spi_nor'. + * @sr1: byte value to be written to the Status Register. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_write_sr1_and_check(struct spi_nor *nor, u8 sr1) +{ + int ret; + + nor->bouncebuf[0] = sr1; + + ret = spi_nor_write_sr(nor, nor->bouncebuf, 1); + if (ret) + return ret; + + ret = spi_nor_read_sr(nor, nor->bouncebuf); + if (ret) + return ret; + + if (nor->bouncebuf[0] != sr1) { + dev_dbg(nor->dev, "SR1: read back test failed\n"); + return -EIO; + } + + return 0; +} + +/** + * spi_nor_write_16bit_sr_and_check() - Write the Status Register 1 and the + * Status Register 2 in one shot. Ensure that the byte written in the Status + * Register 1 match the received value, and that the 16-bit Write did not + * affect what was already in the Status Register 2. + * @nor: pointer to a 'struct spi_nor'. + * @sr1: byte value to be written to the Status Register 1. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_write_16bit_sr_and_check(struct spi_nor *nor, u8 sr1) +{ + int ret; + u8 *sr_cr = nor->bouncebuf; + u8 cr_written; + + /* Make sure we don't overwrite the contents of Status Register 2. */ + if (!(nor->flags & SNOR_F_NO_READ_CR)) { + ret = spi_nor_read_cr(nor, &sr_cr[1]); + if (ret) + return ret; + } else if (nor->params->quad_enable) { + /* + * If the Status Register 2 Read command (35h) is not + * supported, we should at least be sure we don't + * change the value of the SR2 Quad Enable bit. + * + * We can safely assume that when the Quad Enable method is + * set, the value of the QE bit is one, as a consequence of the + * nor->params->quad_enable() call. + * + * We can safely assume that the Quad Enable bit is present in + * the Status Register 2 at BIT(1). According to the JESD216 + * revB standard, BFPT DWORDS[15], bits 22:20, the 16-bit + * Write Status (01h) command is available just for the cases + * in which the QE bit is described in SR2 at BIT(1). + */ + sr_cr[1] = SR2_QUAD_EN_BIT1; + } else { + sr_cr[1] = 0; + } + + sr_cr[0] = sr1; + + ret = spi_nor_write_sr(nor, sr_cr, 2); + if (ret) + return ret; + + if (nor->flags & SNOR_F_NO_READ_CR) + return 0; + + cr_written = sr_cr[1]; + + ret = spi_nor_read_cr(nor, &sr_cr[1]); + if (ret) + return ret; + + if (cr_written != sr_cr[1]) { + dev_dbg(nor->dev, "CR: read back test failed\n"); + return -EIO; + } + + return 0; +} + +/** + * spi_nor_write_16bit_cr_and_check() - Write the Status Register 1 and the + * Configuration Register in one shot. Ensure that the byte written in the + * Configuration Register match the received value, and that the 16-bit Write + * did not affect what was already in the Status Register 1. + * @nor: pointer to a 'struct spi_nor'. + * @cr: byte value to be written to the Configuration Register. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_write_16bit_cr_and_check(struct spi_nor *nor, u8 cr) +{ + int ret; + u8 *sr_cr = nor->bouncebuf; + u8 sr_written; + + /* Keep the current value of the Status Register 1. */ + ret = spi_nor_read_sr(nor, sr_cr); + if (ret) + return ret; + + sr_cr[1] = cr; + + ret = spi_nor_write_sr(nor, sr_cr, 2); + if (ret) + return ret; + + sr_written = sr_cr[0]; + + ret = spi_nor_read_sr(nor, sr_cr); + if (ret) + return ret; + + if (sr_written != sr_cr[0]) { + dev_dbg(nor->dev, "SR: Read back test failed\n"); + return -EIO; + } + + if (nor->flags & SNOR_F_NO_READ_CR) + return 0; + + ret = spi_nor_read_cr(nor, &sr_cr[1]); + if (ret) + return ret; + + if (cr != sr_cr[1]) { + dev_dbg(nor->dev, "CR: read back test failed\n"); + return -EIO; + } + + return 0; +} + +/** + * spi_nor_write_sr_and_check() - Write the Status Register 1 and ensure that + * the byte written match the received value without affecting other bits in the + * Status Register 1 and 2. + * @nor: pointer to a 'struct spi_nor'. + * @sr1: byte value to be written to the Status Register. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_write_sr_and_check(struct spi_nor *nor, u8 sr1) +{ + if (nor->flags & SNOR_F_HAS_16BIT_SR) + return spi_nor_write_16bit_sr_and_check(nor, sr1); + + return spi_nor_write_sr1_and_check(nor, sr1); +} + +/** + * spi_nor_write_sr2() - Write the Status Register 2 using the + * SPINOR_OP_WRSR2 (3eh) command. + * @nor: pointer to 'struct spi_nor'. + * @sr2: pointer to DMA-able buffer to write to the Status Register 2. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_write_sr2(struct spi_nor *nor, const u8 *sr2) +{ + int ret; + + ret = spi_nor_write_enable(nor); + if (ret) + return ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WRSR2, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_OUT(1, sr2, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WRSR2, + sr2, 1); + } + + if (ret) { + dev_dbg(nor->dev, "error %d writing SR2\n", ret); + return ret; + } + + return spi_nor_wait_till_ready(nor); +} + +/** + * spi_nor_read_sr2() - Read the Status Register 2 using the + * SPINOR_OP_RDSR2 (3fh) command. + * @nor: pointer to 'struct spi_nor'. + * @sr2: pointer to DMA-able buffer where the value of the + * Status Register 2 will be written. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_read_sr2(struct spi_nor *nor, u8 *sr2) +{ + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDSR2, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_IN(1, sr2, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->read_reg(nor, SPINOR_OP_RDSR2, + sr2, 1); + } + + if (ret) + dev_dbg(nor->dev, "error %d reading SR2\n", ret); + + return ret; +} + +/** + * spi_nor_erase_chip() - Erase the entire flash memory. + * @nor: pointer to 'struct spi_nor'. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_erase_chip(struct spi_nor *nor) +{ + int ret; + + dev_dbg(nor->dev, " %lldKiB\n", (long long)(nor->mtd.size >> 10)); + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_CHIP_ERASE, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_NO_DATA); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->write_reg(nor, SPINOR_OP_CHIP_ERASE, + NULL, 0); + } + + if (ret) + dev_dbg(nor->dev, "error %d erasing chip\n", ret); + + return ret; +} + +static u8 spi_nor_convert_opcode(u8 opcode, const u8 table[][2], size_t size) +{ + size_t i; + + for (i = 0; i < size; i++) + if (table[i][0] == opcode) + return table[i][1]; + + /* No conversion found, keep input op code. */ + return opcode; +} + +u8 spi_nor_convert_3to4_read(u8 opcode) +{ + static const u8 spi_nor_3to4_read[][2] = { + { SPINOR_OP_READ, SPINOR_OP_READ_4B }, + { SPINOR_OP_READ_FAST, SPINOR_OP_READ_FAST_4B }, + { SPINOR_OP_READ_1_1_2, SPINOR_OP_READ_1_1_2_4B }, + { SPINOR_OP_READ_1_2_2, SPINOR_OP_READ_1_2_2_4B }, + { SPINOR_OP_READ_1_1_4, SPINOR_OP_READ_1_1_4_4B }, + { SPINOR_OP_READ_1_4_4, SPINOR_OP_READ_1_4_4_4B }, + { SPINOR_OP_READ_1_1_8, SPINOR_OP_READ_1_1_8_4B }, + { SPINOR_OP_READ_1_8_8, SPINOR_OP_READ_1_8_8_4B }, + + { SPINOR_OP_READ_1_1_1_DTR, SPINOR_OP_READ_1_1_1_DTR_4B }, + { SPINOR_OP_READ_1_2_2_DTR, SPINOR_OP_READ_1_2_2_DTR_4B }, + { SPINOR_OP_READ_1_4_4_DTR, SPINOR_OP_READ_1_4_4_DTR_4B }, + }; + + return spi_nor_convert_opcode(opcode, spi_nor_3to4_read, + ARRAY_SIZE(spi_nor_3to4_read)); +} + +static u8 spi_nor_convert_3to4_program(u8 opcode) +{ + static const u8 spi_nor_3to4_program[][2] = { + { SPINOR_OP_PP, SPINOR_OP_PP_4B }, + { SPINOR_OP_PP_1_1_4, SPINOR_OP_PP_1_1_4_4B }, + { SPINOR_OP_PP_1_4_4, SPINOR_OP_PP_1_4_4_4B }, + { SPINOR_OP_PP_1_1_8, SPINOR_OP_PP_1_1_8_4B }, + { SPINOR_OP_PP_1_8_8, SPINOR_OP_PP_1_8_8_4B }, + }; + + return spi_nor_convert_opcode(opcode, spi_nor_3to4_program, + ARRAY_SIZE(spi_nor_3to4_program)); +} + +static u8 spi_nor_convert_3to4_erase(u8 opcode) +{ + static const u8 spi_nor_3to4_erase[][2] = { + { SPINOR_OP_BE_4K, SPINOR_OP_BE_4K_4B }, + { SPINOR_OP_BE_32K, SPINOR_OP_BE_32K_4B }, + { SPINOR_OP_SE, SPINOR_OP_SE_4B }, + }; + + return spi_nor_convert_opcode(opcode, spi_nor_3to4_erase, + ARRAY_SIZE(spi_nor_3to4_erase)); +} + +static bool spi_nor_has_uniform_erase(const struct spi_nor *nor) +{ + return !!nor->params->erase_map.uniform_erase_type; +} + +static void spi_nor_set_4byte_opcodes(struct spi_nor *nor) +{ + nor->read_opcode = spi_nor_convert_3to4_read(nor->read_opcode); + nor->program_opcode = spi_nor_convert_3to4_program(nor->program_opcode); + nor->erase_opcode = spi_nor_convert_3to4_erase(nor->erase_opcode); + + if (!spi_nor_has_uniform_erase(nor)) { + struct spi_nor_erase_map *map = &nor->params->erase_map; + struct spi_nor_erase_type *erase; + int i; + + for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { + erase = &map->erase_type[i]; + erase->opcode = + spi_nor_convert_3to4_erase(erase->opcode); + } + } +} + +int spi_nor_lock_and_prep(struct spi_nor *nor) +{ + int ret = 0; + + mutex_lock(&nor->lock); + + if (nor->controller_ops && nor->controller_ops->prepare) { + ret = nor->controller_ops->prepare(nor); + if (ret) { + mutex_unlock(&nor->lock); + return ret; + } + } + return ret; +} + +void spi_nor_unlock_and_unprep(struct spi_nor *nor) +{ + if (nor->controller_ops && nor->controller_ops->unprepare) + nor->controller_ops->unprepare(nor); + mutex_unlock(&nor->lock); +} + +static u32 spi_nor_convert_addr(struct spi_nor *nor, loff_t addr) +{ + if (!nor->params->convert_addr) + return addr; + + return nor->params->convert_addr(nor, addr); +} + +/* + * Initiate the erasure of a single sector + */ +static int spi_nor_erase_sector(struct spi_nor *nor, u32 addr) +{ + int i; + + addr = spi_nor_convert_addr(nor, addr); + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(nor->erase_opcode, 1), + SPI_MEM_OP_ADDR(nor->addr_width, addr, 1), + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_NO_DATA); + + return spi_mem_exec_op(nor->spimem, &op); + } else if (nor->controller_ops->erase) { + return nor->controller_ops->erase(nor, addr); + } + + /* + * Default implementation, if driver doesn't have a specialized HW + * control + */ + for (i = nor->addr_width - 1; i >= 0; i--) { + nor->bouncebuf[i] = addr & 0xff; + addr >>= 8; + } + + return nor->controller_ops->write_reg(nor, nor->erase_opcode, + nor->bouncebuf, nor->addr_width); +} + +/** + * spi_nor_div_by_erase_size() - calculate remainder and update new dividend + * @erase: pointer to a structure that describes a SPI NOR erase type + * @dividend: dividend value + * @remainder: pointer to u32 remainder (will be updated) + * + * Return: the result of the division + */ +static u64 spi_nor_div_by_erase_size(const struct spi_nor_erase_type *erase, + u64 dividend, u32 *remainder) +{ + /* JEDEC JESD216B Standard imposes erase sizes to be power of 2. */ + *remainder = (u32)dividend & erase->size_mask; + return dividend >> erase->size_shift; +} + +/** + * spi_nor_find_best_erase_type() - find the best erase type for the given + * offset in the serial flash memory and the + * number of bytes to erase. The region in + * which the address fits is expected to be + * provided. + * @map: the erase map of the SPI NOR + * @region: pointer to a structure that describes a SPI NOR erase region + * @addr: offset in the serial flash memory + * @len: number of bytes to erase + * + * Return: a pointer to the best fitted erase type, NULL otherwise. + */ +static const struct spi_nor_erase_type * +spi_nor_find_best_erase_type(const struct spi_nor_erase_map *map, + const struct spi_nor_erase_region *region, + u64 addr, u32 len) +{ + const struct spi_nor_erase_type *erase; + u32 rem; + int i; + u8 erase_mask = region->offset & SNOR_ERASE_TYPE_MASK; + + /* + * Erase types are ordered by size, with the smallest erase type at + * index 0. + */ + for (i = SNOR_ERASE_TYPE_MAX - 1; i >= 0; i--) { + /* Does the erase region support the tested erase type? */ + if (!(erase_mask & BIT(i))) + continue; + + erase = &map->erase_type[i]; + + /* Don't erase more than what the user has asked for. */ + if (erase->size > len) + continue; + + /* Alignment is not mandatory for overlaid regions */ + if (region->offset & SNOR_OVERLAID_REGION) + return erase; + + spi_nor_div_by_erase_size(erase, addr, &rem); + if (rem) + continue; + else + return erase; + } + + return NULL; +} + +static u64 spi_nor_region_is_last(const struct spi_nor_erase_region *region) +{ + return region->offset & SNOR_LAST_REGION; +} + +static u64 spi_nor_region_end(const struct spi_nor_erase_region *region) +{ + return (region->offset & ~SNOR_ERASE_FLAGS_MASK) + region->size; +} + +/** + * spi_nor_region_next() - get the next spi nor region + * @region: pointer to a structure that describes a SPI NOR erase region + * + * Return: the next spi nor region or NULL if last region. + */ +struct spi_nor_erase_region * +spi_nor_region_next(struct spi_nor_erase_region *region) +{ + if (spi_nor_region_is_last(region)) + return NULL; + region++; + return region; +} + +/** + * spi_nor_find_erase_region() - find the region of the serial flash memory in + * which the offset fits + * @map: the erase map of the SPI NOR + * @addr: offset in the serial flash memory + * + * Return: a pointer to the spi_nor_erase_region struct, ERR_PTR(-errno) + * otherwise. + */ +static struct spi_nor_erase_region * +spi_nor_find_erase_region(const struct spi_nor_erase_map *map, u64 addr) +{ + struct spi_nor_erase_region *region = map->regions; + u64 region_start = region->offset & ~SNOR_ERASE_FLAGS_MASK; + u64 region_end = region_start + region->size; + + while (addr < region_start || addr >= region_end) { + region = spi_nor_region_next(region); + if (!region) + return ERR_PTR(-EINVAL); + + region_start = region->offset & ~SNOR_ERASE_FLAGS_MASK; + region_end = region_start + region->size; + } + + return region; +} + +/** + * spi_nor_init_erase_cmd() - initialize an erase command + * @region: pointer to a structure that describes a SPI NOR erase region + * @erase: pointer to a structure that describes a SPI NOR erase type + * + * Return: the pointer to the allocated erase command, ERR_PTR(-errno) + * otherwise. + */ +static struct spi_nor_erase_command * +spi_nor_init_erase_cmd(const struct spi_nor_erase_region *region, + const struct spi_nor_erase_type *erase) +{ + struct spi_nor_erase_command *cmd; + + cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); + if (!cmd) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&cmd->list); + cmd->opcode = erase->opcode; + cmd->count = 1; + + if (region->offset & SNOR_OVERLAID_REGION) + cmd->size = region->size; + else + cmd->size = erase->size; + + return cmd; +} + +/** + * spi_nor_destroy_erase_cmd_list() - destroy erase command list + * @erase_list: list of erase commands + */ +static void spi_nor_destroy_erase_cmd_list(struct list_head *erase_list) +{ + struct spi_nor_erase_command *cmd, *next; + + list_for_each_entry_safe(cmd, next, erase_list, list) { + list_del(&cmd->list); + kfree(cmd); + } +} + +/** + * spi_nor_init_erase_cmd_list() - initialize erase command list + * @nor: pointer to a 'struct spi_nor' + * @erase_list: list of erase commands to be executed once we validate that the + * erase can be performed + * @addr: offset in the serial flash memory + * @len: number of bytes to erase + * + * Builds the list of best fitted erase commands and verifies if the erase can + * be performed. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_init_erase_cmd_list(struct spi_nor *nor, + struct list_head *erase_list, + u64 addr, u32 len) +{ + const struct spi_nor_erase_map *map = &nor->params->erase_map; + const struct spi_nor_erase_type *erase, *prev_erase = NULL; + struct spi_nor_erase_region *region; + struct spi_nor_erase_command *cmd = NULL; + u64 region_end; + int ret = -EINVAL; + + region = spi_nor_find_erase_region(map, addr); + if (IS_ERR(region)) + return PTR_ERR(region); + + region_end = spi_nor_region_end(region); + + while (len) { + erase = spi_nor_find_best_erase_type(map, region, addr, len); + if (!erase) + goto destroy_erase_cmd_list; + + if (prev_erase != erase || + region->offset & SNOR_OVERLAID_REGION) { + cmd = spi_nor_init_erase_cmd(region, erase); + if (IS_ERR(cmd)) { + ret = PTR_ERR(cmd); + goto destroy_erase_cmd_list; + } + + list_add_tail(&cmd->list, erase_list); + } else { + cmd->count++; + } + + addr += cmd->size; + len -= cmd->size; + + if (len && addr >= region_end) { + region = spi_nor_region_next(region); + if (!region) + goto destroy_erase_cmd_list; + region_end = spi_nor_region_end(region); + } + + prev_erase = erase; + } + + return 0; + +destroy_erase_cmd_list: + spi_nor_destroy_erase_cmd_list(erase_list); + return ret; +} + +/** + * spi_nor_erase_multi_sectors() - perform a non-uniform erase + * @nor: pointer to a 'struct spi_nor' + * @addr: offset in the serial flash memory + * @len: number of bytes to erase + * + * Build a list of best fitted erase commands and execute it once we validate + * that the erase can be performed. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_erase_multi_sectors(struct spi_nor *nor, u64 addr, u32 len) +{ + LIST_HEAD(erase_list); + struct spi_nor_erase_command *cmd, *next; + int ret; + + ret = spi_nor_init_erase_cmd_list(nor, &erase_list, addr, len); + if (ret) + return ret; + + list_for_each_entry_safe(cmd, next, &erase_list, list) { + nor->erase_opcode = cmd->opcode; + while (cmd->count) { + ret = spi_nor_write_enable(nor); + if (ret) + goto destroy_erase_cmd_list; + + ret = spi_nor_erase_sector(nor, addr); + if (ret) + goto destroy_erase_cmd_list; + + addr += cmd->size; + cmd->count--; + + ret = spi_nor_wait_till_ready(nor); + if (ret) + goto destroy_erase_cmd_list; + } + list_del(&cmd->list); + kfree(cmd); + } + + return 0; + +destroy_erase_cmd_list: + spi_nor_destroy_erase_cmd_list(&erase_list); + return ret; +} + +/* + * Erase an address range on the nor chip. The address range may extend + * one or more erase sectors. Return an error is there is a problem erasing. + */ +static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + u32 addr, len; + uint32_t rem; + int ret; + + dev_dbg(nor->dev, "at 0x%llx, len %lld\n", (long long)instr->addr, + (long long)instr->len); + + if (spi_nor_has_uniform_erase(nor)) { + div_u64_rem(instr->len, mtd->erasesize, &rem); + if (rem) + return -EINVAL; + } + + addr = instr->addr; + len = instr->len; + + ret = spi_nor_lock_and_prep(nor); + if (ret) + return ret; + + /* whole-chip erase? */ + if (len == mtd->size && !(nor->flags & SNOR_F_NO_OP_CHIP_ERASE)) { + unsigned long timeout; + + ret = spi_nor_write_enable(nor); + if (ret) + goto erase_err; + + ret = spi_nor_erase_chip(nor); + if (ret) + goto erase_err; + + /* + * Scale the timeout linearly with the size of the flash, with + * a minimum calibrated to an old 2MB flash. We could try to + * pull these from CFI/SFDP, but these values should be good + * enough for now. + */ + timeout = max(CHIP_ERASE_2MB_READY_WAIT_JIFFIES, + CHIP_ERASE_2MB_READY_WAIT_JIFFIES * + (unsigned long)(mtd->size / SZ_2M)); + ret = spi_nor_wait_till_ready_with_timeout(nor, timeout); + if (ret) + goto erase_err; + + /* REVISIT in some cases we could speed up erasing large regions + * by using SPINOR_OP_SE instead of SPINOR_OP_BE_4K. We may have set up + * to use "small sector erase", but that's not always optimal. + */ + + /* "sector"-at-a-time erase */ + } else if (spi_nor_has_uniform_erase(nor)) { + while (len) { + ret = spi_nor_write_enable(nor); + if (ret) + goto erase_err; + + ret = spi_nor_erase_sector(nor, addr); + if (ret) + goto erase_err; + + addr += mtd->erasesize; + len -= mtd->erasesize; + + ret = spi_nor_wait_till_ready(nor); + if (ret) + goto erase_err; + } + + /* erase multiple sectors */ + } else { + ret = spi_nor_erase_multi_sectors(nor, addr, len); + if (ret) + goto erase_err; + } + + ret = spi_nor_write_disable(nor); + +erase_err: + spi_nor_unlock_and_unprep(nor); + + return ret; +} + +static u8 spi_nor_get_sr_bp_mask(struct spi_nor *nor) +{ + u8 mask = SR_BP2 | SR_BP1 | SR_BP0; + + if (nor->flags & SNOR_F_HAS_SR_BP3_BIT6) + return mask | SR_BP3_BIT6; + + if (nor->flags & SNOR_F_HAS_4BIT_BP) + return mask | SR_BP3; + + return mask; +} + +static u8 spi_nor_get_sr_tb_mask(struct spi_nor *nor) +{ + if (nor->flags & SNOR_F_HAS_SR_TB_BIT6) + return SR_TB_BIT6; + else + return SR_TB_BIT5; +} + +static u64 spi_nor_get_min_prot_length_sr(struct spi_nor *nor) +{ + unsigned int bp_slots, bp_slots_needed; + u8 mask = spi_nor_get_sr_bp_mask(nor); + + /* Reserved one for "protect none" and one for "protect all". */ + bp_slots = (1 << hweight8(mask)) - 2; + bp_slots_needed = ilog2(nor->info->n_sectors); + + if (bp_slots_needed > bp_slots) + return nor->info->sector_size << + (bp_slots_needed - bp_slots); + else + return nor->info->sector_size; +} + +static void spi_nor_get_locked_range_sr(struct spi_nor *nor, u8 sr, loff_t *ofs, + uint64_t *len) +{ + struct mtd_info *mtd = &nor->mtd; + u64 min_prot_len; + u8 mask = spi_nor_get_sr_bp_mask(nor); + u8 tb_mask = spi_nor_get_sr_tb_mask(nor); + u8 bp, val = sr & mask; + + if (nor->flags & SNOR_F_HAS_SR_BP3_BIT6 && val & SR_BP3_BIT6) + val = (val & ~SR_BP3_BIT6) | SR_BP3; + + bp = val >> SR_BP_SHIFT; + + if (!bp) { + /* No protection */ + *ofs = 0; + *len = 0; + return; + } + + min_prot_len = spi_nor_get_min_prot_length_sr(nor); + *len = min_prot_len << (bp - 1); + + if (*len > mtd->size) + *len = mtd->size; + + if (nor->flags & SNOR_F_HAS_SR_TB && sr & tb_mask) + *ofs = 0; + else + *ofs = mtd->size - *len; +} + +/* + * Return 1 if the entire region is locked (if @locked is true) or unlocked (if + * @locked is false); 0 otherwise + */ +static int spi_nor_check_lock_status_sr(struct spi_nor *nor, loff_t ofs, + uint64_t len, u8 sr, bool locked) +{ + loff_t lock_offs; + uint64_t lock_len; + + if (!len) + return 1; + + spi_nor_get_locked_range_sr(nor, sr, &lock_offs, &lock_len); + + if (locked) + /* Requested range is a sub-range of locked range */ + return (ofs + len <= lock_offs + lock_len) && (ofs >= lock_offs); + else + /* Requested range does not overlap with locked range */ + return (ofs >= lock_offs + lock_len) || (ofs + len <= lock_offs); +} + +static int spi_nor_is_locked_sr(struct spi_nor *nor, loff_t ofs, uint64_t len, + u8 sr) +{ + return spi_nor_check_lock_status_sr(nor, ofs, len, sr, true); +} + +static int spi_nor_is_unlocked_sr(struct spi_nor *nor, loff_t ofs, uint64_t len, + u8 sr) +{ + return spi_nor_check_lock_status_sr(nor, ofs, len, sr, false); +} + +/* + * Lock a region of the flash. Compatible with ST Micro and similar flash. + * Supports the block protection bits BP{0,1,2}/BP{0,1,2,3} in the status + * register + * (SR). Does not support these features found in newer SR bitfields: + * - SEC: sector/block protect - only handle SEC=0 (block protect) + * - CMP: complement protect - only support CMP=0 (range is not complemented) + * + * Support for the following is provided conditionally for some flash: + * - TB: top/bottom protect + * + * Sample table portion for 8MB flash (Winbond w25q64fw): + * + * SEC | TB | BP2 | BP1 | BP0 | Prot Length | Protected Portion + * -------------------------------------------------------------------------- + * X | X | 0 | 0 | 0 | NONE | NONE + * 0 | 0 | 0 | 0 | 1 | 128 KB | Upper 1/64 + * 0 | 0 | 0 | 1 | 0 | 256 KB | Upper 1/32 + * 0 | 0 | 0 | 1 | 1 | 512 KB | Upper 1/16 + * 0 | 0 | 1 | 0 | 0 | 1 MB | Upper 1/8 + * 0 | 0 | 1 | 0 | 1 | 2 MB | Upper 1/4 + * 0 | 0 | 1 | 1 | 0 | 4 MB | Upper 1/2 + * X | X | 1 | 1 | 1 | 8 MB | ALL + * ------|-------|-------|-------|-------|---------------|------------------- + * 0 | 1 | 0 | 0 | 1 | 128 KB | Lower 1/64 + * 0 | 1 | 0 | 1 | 0 | 256 KB | Lower 1/32 + * 0 | 1 | 0 | 1 | 1 | 512 KB | Lower 1/16 + * 0 | 1 | 1 | 0 | 0 | 1 MB | Lower 1/8 + * 0 | 1 | 1 | 0 | 1 | 2 MB | Lower 1/4 + * 0 | 1 | 1 | 1 | 0 | 4 MB | Lower 1/2 + * + * Returns negative on errors, 0 on success. + */ +static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, uint64_t len) +{ + struct mtd_info *mtd = &nor->mtd; + u64 min_prot_len; + int ret, status_old, status_new; + u8 mask = spi_nor_get_sr_bp_mask(nor); + u8 tb_mask = spi_nor_get_sr_tb_mask(nor); + u8 pow, val; + loff_t lock_len; + bool can_be_top = true, can_be_bottom = nor->flags & SNOR_F_HAS_SR_TB; + bool use_top; + + ret = spi_nor_read_sr(nor, nor->bouncebuf); + if (ret) + return ret; + + status_old = nor->bouncebuf[0]; + + /* If nothing in our range is unlocked, we don't need to do anything */ + if (spi_nor_is_locked_sr(nor, ofs, len, status_old)) + return 0; + + /* If anything below us is unlocked, we can't use 'bottom' protection */ + if (!spi_nor_is_locked_sr(nor, 0, ofs, status_old)) + can_be_bottom = false; + + /* If anything above us is unlocked, we can't use 'top' protection */ + if (!spi_nor_is_locked_sr(nor, ofs + len, mtd->size - (ofs + len), + status_old)) + can_be_top = false; + + if (!can_be_bottom && !can_be_top) + return -EINVAL; + + /* Prefer top, if both are valid */ + use_top = can_be_top; + + /* lock_len: length of region that should end up locked */ + if (use_top) + lock_len = mtd->size - ofs; + else + lock_len = ofs + len; + + if (lock_len == mtd->size) { + val = mask; + } else { + min_prot_len = spi_nor_get_min_prot_length_sr(nor); + pow = ilog2(lock_len) - ilog2(min_prot_len) + 1; + val = pow << SR_BP_SHIFT; + + if (nor->flags & SNOR_F_HAS_SR_BP3_BIT6 && val & SR_BP3) + val = (val & ~SR_BP3) | SR_BP3_BIT6; + + if (val & ~mask) + return -EINVAL; + + /* Don't "lock" with no region! */ + if (!(val & mask)) + return -EINVAL; + } + + status_new = (status_old & ~mask & ~tb_mask) | val; + + /* Disallow further writes if WP pin is asserted */ + status_new |= SR_SRWD; + + if (!use_top) + status_new |= tb_mask; + + /* Don't bother if they're the same */ + if (status_new == status_old) + return 0; + + /* Only modify protection if it will not unlock other areas */ + if ((status_new & mask) < (status_old & mask)) + return -EINVAL; + + return spi_nor_write_sr_and_check(nor, status_new); +} + +/* + * Unlock a region of the flash. See spi_nor_sr_lock() for more info + * + * Returns negative on errors, 0 on success. + */ +static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len) +{ + struct mtd_info *mtd = &nor->mtd; + u64 min_prot_len; + int ret, status_old, status_new; + u8 mask = spi_nor_get_sr_bp_mask(nor); + u8 tb_mask = spi_nor_get_sr_tb_mask(nor); + u8 pow, val; + loff_t lock_len; + bool can_be_top = true, can_be_bottom = nor->flags & SNOR_F_HAS_SR_TB; + bool use_top; + + ret = spi_nor_read_sr(nor, nor->bouncebuf); + if (ret) + return ret; + + status_old = nor->bouncebuf[0]; + + /* If nothing in our range is locked, we don't need to do anything */ + if (spi_nor_is_unlocked_sr(nor, ofs, len, status_old)) + return 0; + + /* If anything below us is locked, we can't use 'top' protection */ + if (!spi_nor_is_unlocked_sr(nor, 0, ofs, status_old)) + can_be_top = false; + + /* If anything above us is locked, we can't use 'bottom' protection */ + if (!spi_nor_is_unlocked_sr(nor, ofs + len, mtd->size - (ofs + len), + status_old)) + can_be_bottom = false; + + if (!can_be_bottom && !can_be_top) + return -EINVAL; + + /* Prefer top, if both are valid */ + use_top = can_be_top; + + /* lock_len: length of region that should remain locked */ + if (use_top) + lock_len = mtd->size - (ofs + len); + else + lock_len = ofs; + + if (lock_len == 0) { + val = 0; /* fully unlocked */ + } else { + min_prot_len = spi_nor_get_min_prot_length_sr(nor); + pow = ilog2(lock_len) - ilog2(min_prot_len) + 1; + val = pow << SR_BP_SHIFT; + + if (nor->flags & SNOR_F_HAS_SR_BP3_BIT6 && val & SR_BP3) + val = (val & ~SR_BP3) | SR_BP3_BIT6; + + /* Some power-of-two sizes are not supported */ + if (val & ~mask) + return -EINVAL; + } + + status_new = (status_old & ~mask & ~tb_mask) | val; + + /* Don't protect status register if we're fully unlocked */ + if (lock_len == 0) + status_new &= ~SR_SRWD; + + if (!use_top) + status_new |= tb_mask; + + /* Don't bother if they're the same */ + if (status_new == status_old) + return 0; + + /* Only modify protection if it will not lock other areas */ + if ((status_new & mask) > (status_old & mask)) + return -EINVAL; + + return spi_nor_write_sr_and_check(nor, status_new); +} + +/* + * Check if a region of the flash is (completely) locked. See spi_nor_sr_lock() + * for more info. + * + * Returns 1 if entire region is locked, 0 if any portion is unlocked, and + * negative on errors. + */ +static int spi_nor_sr_is_locked(struct spi_nor *nor, loff_t ofs, uint64_t len) +{ + int ret; + + ret = spi_nor_read_sr(nor, nor->bouncebuf); + if (ret) + return ret; + + return spi_nor_is_locked_sr(nor, ofs, len, nor->bouncebuf[0]); +} + +static const struct spi_nor_locking_ops spi_nor_sr_locking_ops = { + .lock = spi_nor_sr_lock, + .unlock = spi_nor_sr_unlock, + .is_locked = spi_nor_sr_is_locked, +}; + +static int spi_nor_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + int ret; + + ret = spi_nor_lock_and_prep(nor); + if (ret) + return ret; + + ret = nor->params->locking_ops->lock(nor, ofs, len); + + spi_nor_unlock_and_unprep(nor); + return ret; +} + +static int spi_nor_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + int ret; + + ret = spi_nor_lock_and_prep(nor); + if (ret) + return ret; + + ret = nor->params->locking_ops->unlock(nor, ofs, len); + + spi_nor_unlock_and_unprep(nor); + return ret; +} + +static int spi_nor_is_locked(struct mtd_info *mtd, loff_t ofs, uint64_t len) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + int ret; + + ret = spi_nor_lock_and_prep(nor); + if (ret) + return ret; + + ret = nor->params->locking_ops->is_locked(nor, ofs, len); + + spi_nor_unlock_and_unprep(nor); + return ret; +} + +/** + * spi_nor_sr1_bit6_quad_enable() - Set the Quad Enable BIT(6) in the Status + * Register 1. + * @nor: pointer to a 'struct spi_nor' + * + * Bit 6 of the Status Register 1 is the QE bit for Macronix like QSPI memories. + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_sr1_bit6_quad_enable(struct spi_nor *nor) +{ + int ret; + + ret = spi_nor_read_sr(nor, nor->bouncebuf); + if (ret) + return ret; + + if (nor->bouncebuf[0] & SR1_QUAD_EN_BIT6) + return 0; + + nor->bouncebuf[0] |= SR1_QUAD_EN_BIT6; + + return spi_nor_write_sr1_and_check(nor, nor->bouncebuf[0]); +} + +/** + * spi_nor_sr2_bit1_quad_enable() - set the Quad Enable BIT(1) in the Status + * Register 2. + * @nor: pointer to a 'struct spi_nor'. + * + * Bit 1 of the Status Register 2 is the QE bit for Spansion like QSPI memories. + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_sr2_bit1_quad_enable(struct spi_nor *nor) +{ + int ret; + + if (nor->flags & SNOR_F_NO_READ_CR) + return spi_nor_write_16bit_cr_and_check(nor, SR2_QUAD_EN_BIT1); + + ret = spi_nor_read_cr(nor, nor->bouncebuf); + if (ret) + return ret; + + if (nor->bouncebuf[0] & SR2_QUAD_EN_BIT1) + return 0; + + nor->bouncebuf[0] |= SR2_QUAD_EN_BIT1; + + return spi_nor_write_16bit_cr_and_check(nor, nor->bouncebuf[0]); +} + +/** + * spi_nor_sr2_bit7_quad_enable() - set QE bit in Status Register 2. + * @nor: pointer to a 'struct spi_nor' + * + * Set the Quad Enable (QE) bit in the Status Register 2. + * + * This is one of the procedures to set the QE bit described in the SFDP + * (JESD216 rev B) specification but no manufacturer using this procedure has + * been identified yet, hence the name of the function. + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_sr2_bit7_quad_enable(struct spi_nor *nor) +{ + u8 *sr2 = nor->bouncebuf; + int ret; + u8 sr2_written; + + /* Check current Quad Enable bit value. */ + ret = spi_nor_read_sr2(nor, sr2); + if (ret) + return ret; + if (*sr2 & SR2_QUAD_EN_BIT7) + return 0; + + /* Update the Quad Enable bit. */ + *sr2 |= SR2_QUAD_EN_BIT7; + + ret = spi_nor_write_sr2(nor, sr2); + if (ret) + return ret; + + sr2_written = *sr2; + + /* Read back and check it. */ + ret = spi_nor_read_sr2(nor, sr2); + if (ret) + return ret; + + if (*sr2 != sr2_written) { + dev_dbg(nor->dev, "SR2: Read back test failed\n"); + return -EIO; + } + + return 0; +} + +static const struct spi_nor_manufacturer *manufacturers[] = { + &spi_nor_atmel, + &spi_nor_catalyst, + &spi_nor_eon, + &spi_nor_esmt, + &spi_nor_everspin, + &spi_nor_fujitsu, + &spi_nor_gigadevice, + &spi_nor_intel, + &spi_nor_issi, + &spi_nor_macronix, + &spi_nor_micron, + &spi_nor_st, + &spi_nor_spansion, + &spi_nor_sst, + &spi_nor_winbond, + &spi_nor_xilinx, + &spi_nor_xmc, +}; + +static const struct flash_info * +spi_nor_search_part_by_id(const struct flash_info *parts, unsigned int nparts, + const u8 *id) +{ + unsigned int i; + + for (i = 0; i < nparts; i++) { + if (parts[i].id_len && + !memcmp(parts[i].id, id, parts[i].id_len)) + return &parts[i]; + } + + return NULL; +} + +static const struct flash_info *spi_nor_read_id(struct spi_nor *nor) +{ + const struct flash_info *info; + u8 *id = nor->bouncebuf; + unsigned int i; + int ret; + + if (nor->spimem) { + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDID, 1), + SPI_MEM_OP_NO_ADDR, + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_IN(SPI_NOR_MAX_ID_LEN, id, 1)); + + ret = spi_mem_exec_op(nor->spimem, &op); + } else { + ret = nor->controller_ops->read_reg(nor, SPINOR_OP_RDID, id, + SPI_NOR_MAX_ID_LEN); + } + if (ret) { + dev_dbg(nor->dev, "error %d reading JEDEC ID\n", ret); + return ERR_PTR(ret); + } + + for (i = 0; i < ARRAY_SIZE(manufacturers); i++) { + info = spi_nor_search_part_by_id(manufacturers[i]->parts, + manufacturers[i]->nparts, + id); + if (info) { + nor->manufacturer = manufacturers[i]; + return info; + } + } + + dev_err(nor->dev, "unrecognized JEDEC id bytes: %*ph\n", + SPI_NOR_MAX_ID_LEN, id); + return ERR_PTR(-ENODEV); +} + +static int spi_nor_read(struct mtd_info *mtd, loff_t from, size_t len, + size_t *retlen, u_char *buf) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + ssize_t ret; + + dev_dbg(nor->dev, "from 0x%08x, len %zd\n", (u32)from, len); + + ret = spi_nor_lock_and_prep(nor); + if (ret) + return ret; + + while (len) { + loff_t addr = from; + + addr = spi_nor_convert_addr(nor, addr); + + ret = spi_nor_read_data(nor, addr, len, buf); + if (ret == 0) { + /* We shouldn't see 0-length reads */ + ret = -EIO; + goto read_err; + } + if (ret < 0) + goto read_err; + + WARN_ON(ret > len); + *retlen += ret; + buf += ret; + from += ret; + len -= ret; + } + ret = 0; + +read_err: + spi_nor_unlock_and_unprep(nor); + return ret; +} + +/* + * Write an address range to the nor chip. Data must be written in + * FLASH_PAGESIZE chunks. The address range may be any size provided + * it is within the physical boundaries. + */ +static int spi_nor_write(struct mtd_info *mtd, loff_t to, size_t len, + size_t *retlen, const u_char *buf) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + size_t page_offset, page_remain, i; + ssize_t ret; + + dev_dbg(nor->dev, "to 0x%08x, len %zd\n", (u32)to, len); + + ret = spi_nor_lock_and_prep(nor); + if (ret) + return ret; + + for (i = 0; i < len; ) { + ssize_t written; + loff_t addr = to + i; + + /* + * If page_size is a power of two, the offset can be quickly + * calculated with an AND operation. On the other cases we + * need to do a modulus operation (more expensive). + * Power of two numbers have only one bit set and we can use + * the instruction hweight32 to detect if we need to do a + * modulus (do_div()) or not. + */ + if (hweight32(nor->page_size) == 1) { + page_offset = addr & (nor->page_size - 1); + } else { + uint64_t aux = addr; + + page_offset = do_div(aux, nor->page_size); + } + /* the size of data remaining on the first page */ + page_remain = min_t(size_t, + nor->page_size - page_offset, len - i); + + addr = spi_nor_convert_addr(nor, addr); + + ret = spi_nor_write_enable(nor); + if (ret) + goto write_err; + + ret = spi_nor_write_data(nor, addr, page_remain, buf + i); + if (ret < 0) + goto write_err; + written = ret; + + ret = spi_nor_wait_till_ready(nor); + if (ret) + goto write_err; + *retlen += written; + i += written; + } + +write_err: + spi_nor_unlock_and_unprep(nor); + return ret; +} + +static int spi_nor_check(struct spi_nor *nor) +{ + if (!nor->dev || + (!nor->spimem && !nor->controller_ops) || + (!nor->spimem && nor->controller_ops && + (!nor->controller_ops->read || + !nor->controller_ops->write || + !nor->controller_ops->read_reg || + !nor->controller_ops->write_reg))) { + pr_err("spi-nor: please fill all the necessary fields!\n"); + return -EINVAL; + } + + if (nor->spimem && nor->controller_ops) { + dev_err(nor->dev, "nor->spimem and nor->controller_ops are mutually exclusive, please set just one of them.\n"); + return -EINVAL; + } + + return 0; +} + +static void +spi_nor_set_read_settings(struct spi_nor_read_command *read, + u8 num_mode_clocks, + u8 num_wait_states, + u8 opcode, + enum spi_nor_protocol proto) +{ + read->num_mode_clocks = num_mode_clocks; + read->num_wait_states = num_wait_states; + read->opcode = opcode; + read->proto = proto; +} + +void spi_nor_set_pp_settings(struct spi_nor_pp_command *pp, u8 opcode, + enum spi_nor_protocol proto) +{ + pp->opcode = opcode; + pp->proto = proto; +} + +static int spi_nor_hwcaps2cmd(u32 hwcaps, const int table[][2], size_t size) +{ + size_t i; + + for (i = 0; i < size; i++) + if (table[i][0] == (int)hwcaps) + return table[i][1]; + + return -EINVAL; +} + +int spi_nor_hwcaps_read2cmd(u32 hwcaps) +{ + static const int hwcaps_read2cmd[][2] = { + { SNOR_HWCAPS_READ, SNOR_CMD_READ }, + { SNOR_HWCAPS_READ_FAST, SNOR_CMD_READ_FAST }, + { SNOR_HWCAPS_READ_1_1_1_DTR, SNOR_CMD_READ_1_1_1_DTR }, + { SNOR_HWCAPS_READ_1_1_2, SNOR_CMD_READ_1_1_2 }, + { SNOR_HWCAPS_READ_1_2_2, SNOR_CMD_READ_1_2_2 }, + { SNOR_HWCAPS_READ_2_2_2, SNOR_CMD_READ_2_2_2 }, + { SNOR_HWCAPS_READ_1_2_2_DTR, SNOR_CMD_READ_1_2_2_DTR }, + { SNOR_HWCAPS_READ_1_1_4, SNOR_CMD_READ_1_1_4 }, + { SNOR_HWCAPS_READ_1_4_4, SNOR_CMD_READ_1_4_4 }, + { SNOR_HWCAPS_READ_4_4_4, SNOR_CMD_READ_4_4_4 }, + { SNOR_HWCAPS_READ_1_4_4_DTR, SNOR_CMD_READ_1_4_4_DTR }, + { SNOR_HWCAPS_READ_1_1_8, SNOR_CMD_READ_1_1_8 }, + { SNOR_HWCAPS_READ_1_8_8, SNOR_CMD_READ_1_8_8 }, + { SNOR_HWCAPS_READ_8_8_8, SNOR_CMD_READ_8_8_8 }, + { SNOR_HWCAPS_READ_1_8_8_DTR, SNOR_CMD_READ_1_8_8_DTR }, + }; + + return spi_nor_hwcaps2cmd(hwcaps, hwcaps_read2cmd, + ARRAY_SIZE(hwcaps_read2cmd)); +} + +static int spi_nor_hwcaps_pp2cmd(u32 hwcaps) +{ + static const int hwcaps_pp2cmd[][2] = { + { SNOR_HWCAPS_PP, SNOR_CMD_PP }, + { SNOR_HWCAPS_PP_1_1_4, SNOR_CMD_PP_1_1_4 }, + { SNOR_HWCAPS_PP_1_4_4, SNOR_CMD_PP_1_4_4 }, + { SNOR_HWCAPS_PP_4_4_4, SNOR_CMD_PP_4_4_4 }, + { SNOR_HWCAPS_PP_1_1_8, SNOR_CMD_PP_1_1_8 }, + { SNOR_HWCAPS_PP_1_8_8, SNOR_CMD_PP_1_8_8 }, + { SNOR_HWCAPS_PP_8_8_8, SNOR_CMD_PP_8_8_8 }, + }; + + return spi_nor_hwcaps2cmd(hwcaps, hwcaps_pp2cmd, + ARRAY_SIZE(hwcaps_pp2cmd)); +} + +/** + * spi_nor_spimem_check_op - check if the operation is supported + * by controller + *@nor: pointer to a 'struct spi_nor' + *@op: pointer to op template to be checked + * + * Returns 0 if operation is supported, -ENOTSUPP otherwise. + */ +static int spi_nor_spimem_check_op(struct spi_nor *nor, + struct spi_mem_op *op) +{ + /* + * First test with 4 address bytes. The opcode itself might + * be a 3B addressing opcode but we don't care, because + * SPI controller implementation should not check the opcode, + * but just the sequence. + */ + op->addr.nbytes = 4; + if (!spi_mem_supports_op(nor->spimem, op)) { + if (nor->mtd.size > SZ_16M) + return -ENOTSUPP; + + /* If flash size <= 16MB, 3 address bytes are sufficient */ + op->addr.nbytes = 3; + if (!spi_mem_supports_op(nor->spimem, op)) + return -ENOTSUPP; + } + + return 0; +} + +/** + * spi_nor_spimem_check_readop - check if the read op is supported + * by controller + *@nor: pointer to a 'struct spi_nor' + *@read: pointer to op template to be checked + * + * Returns 0 if operation is supported, -ENOTSUPP otherwise. + */ +static int spi_nor_spimem_check_readop(struct spi_nor *nor, + const struct spi_nor_read_command *read) +{ + struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(read->opcode, 1), + SPI_MEM_OP_ADDR(3, 0, 1), + SPI_MEM_OP_DUMMY(0, 1), + SPI_MEM_OP_DATA_IN(0, NULL, 1)); + + op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(read->proto); + op.addr.buswidth = spi_nor_get_protocol_addr_nbits(read->proto); + op.data.buswidth = spi_nor_get_protocol_data_nbits(read->proto); + op.dummy.buswidth = op.addr.buswidth; + op.dummy.nbytes = (read->num_mode_clocks + read->num_wait_states) * + op.dummy.buswidth / 8; + + return spi_nor_spimem_check_op(nor, &op); +} + +/** + * spi_nor_spimem_check_pp - check if the page program op is supported + * by controller + *@nor: pointer to a 'struct spi_nor' + *@pp: pointer to op template to be checked + * + * Returns 0 if operation is supported, -ENOTSUPP otherwise. + */ +static int spi_nor_spimem_check_pp(struct spi_nor *nor, + const struct spi_nor_pp_command *pp) +{ + struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(pp->opcode, 1), + SPI_MEM_OP_ADDR(3, 0, 1), + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_OUT(0, NULL, 1)); + + op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(pp->proto); + op.addr.buswidth = spi_nor_get_protocol_addr_nbits(pp->proto); + op.data.buswidth = spi_nor_get_protocol_data_nbits(pp->proto); + + return spi_nor_spimem_check_op(nor, &op); +} + +/** + * spi_nor_spimem_adjust_hwcaps - Find optimal Read/Write protocol + * based on SPI controller capabilities + * @nor: pointer to a 'struct spi_nor' + * @hwcaps: pointer to resulting capabilities after adjusting + * according to controller and flash's capability + */ +static void +spi_nor_spimem_adjust_hwcaps(struct spi_nor *nor, u32 *hwcaps) +{ + struct spi_nor_flash_parameter *params = nor->params; + unsigned int cap; + + /* DTR modes are not supported yet, mask them all. */ + *hwcaps &= ~SNOR_HWCAPS_DTR; + + /* X-X-X modes are not supported yet, mask them all. */ + *hwcaps &= ~SNOR_HWCAPS_X_X_X; + + for (cap = 0; cap < sizeof(*hwcaps) * BITS_PER_BYTE; cap++) { + int rdidx, ppidx; + + if (!(*hwcaps & BIT(cap))) + continue; + + rdidx = spi_nor_hwcaps_read2cmd(BIT(cap)); + if (rdidx >= 0 && + spi_nor_spimem_check_readop(nor, ¶ms->reads[rdidx])) + *hwcaps &= ~BIT(cap); + + ppidx = spi_nor_hwcaps_pp2cmd(BIT(cap)); + if (ppidx < 0) + continue; + + if (spi_nor_spimem_check_pp(nor, + ¶ms->page_programs[ppidx])) + *hwcaps &= ~BIT(cap); + } +} + +/** + * spi_nor_set_erase_type() - set a SPI NOR erase type + * @erase: pointer to a structure that describes a SPI NOR erase type + * @size: the size of the sector/block erased by the erase type + * @opcode: the SPI command op code to erase the sector/block + */ +void spi_nor_set_erase_type(struct spi_nor_erase_type *erase, u32 size, + u8 opcode) +{ + erase->size = size; + erase->opcode = opcode; + /* JEDEC JESD216B Standard imposes erase sizes to be power of 2. */ + erase->size_shift = ffs(erase->size) - 1; + erase->size_mask = (1 << erase->size_shift) - 1; +} + +/** + * spi_nor_init_uniform_erase_map() - Initialize uniform erase map + * @map: the erase map of the SPI NOR + * @erase_mask: bitmask encoding erase types that can erase the entire + * flash memory + * @flash_size: the spi nor flash memory size + */ +void spi_nor_init_uniform_erase_map(struct spi_nor_erase_map *map, + u8 erase_mask, u64 flash_size) +{ + /* Offset 0 with erase_mask and SNOR_LAST_REGION bit set */ + map->uniform_region.offset = (erase_mask & SNOR_ERASE_TYPE_MASK) | + SNOR_LAST_REGION; + map->uniform_region.size = flash_size; + map->regions = &map->uniform_region; + map->uniform_erase_type = erase_mask; +} + +int spi_nor_post_bfpt_fixups(struct spi_nor *nor, + const struct sfdp_parameter_header *bfpt_header, + const struct sfdp_bfpt *bfpt, + struct spi_nor_flash_parameter *params) +{ + int ret; + + if (nor->manufacturer && nor->manufacturer->fixups && + nor->manufacturer->fixups->post_bfpt) { + ret = nor->manufacturer->fixups->post_bfpt(nor, bfpt_header, + bfpt, params); + if (ret) + return ret; + } + + if (nor->info->fixups && nor->info->fixups->post_bfpt) + return nor->info->fixups->post_bfpt(nor, bfpt_header, bfpt, + params); + + return 0; +} + +static int spi_nor_select_read(struct spi_nor *nor, + u32 shared_hwcaps) +{ + int cmd, best_match = fls(shared_hwcaps & SNOR_HWCAPS_READ_MASK) - 1; + const struct spi_nor_read_command *read; + + if (best_match < 0) + return -EINVAL; + + cmd = spi_nor_hwcaps_read2cmd(BIT(best_match)); + if (cmd < 0) + return -EINVAL; + + read = &nor->params->reads[cmd]; + nor->read_opcode = read->opcode; + nor->read_proto = read->proto; + + /* + * In the spi-nor framework, we don't need to make the difference + * between mode clock cycles and wait state clock cycles. + * Indeed, the value of the mode clock cycles is used by a QSPI + * flash memory to know whether it should enter or leave its 0-4-4 + * (Continuous Read / XIP) mode. + * eXecution In Place is out of the scope of the mtd sub-system. + * Hence we choose to merge both mode and wait state clock cycles + * into the so called dummy clock cycles. + */ + nor->read_dummy = read->num_mode_clocks + read->num_wait_states; + return 0; +} + +static int spi_nor_select_pp(struct spi_nor *nor, + u32 shared_hwcaps) +{ + int cmd, best_match = fls(shared_hwcaps & SNOR_HWCAPS_PP_MASK) - 1; + const struct spi_nor_pp_command *pp; + + if (best_match < 0) + return -EINVAL; + + cmd = spi_nor_hwcaps_pp2cmd(BIT(best_match)); + if (cmd < 0) + return -EINVAL; + + pp = &nor->params->page_programs[cmd]; + nor->program_opcode = pp->opcode; + nor->write_proto = pp->proto; + return 0; +} + +/** + * spi_nor_select_uniform_erase() - select optimum uniform erase type + * @map: the erase map of the SPI NOR + * @wanted_size: the erase type size to search for. Contains the value of + * info->sector_size or of the "small sector" size in case + * CONFIG_MTD_SPI_NOR_USE_4K_SECTORS is defined. + * + * Once the optimum uniform sector erase command is found, disable all the + * other. + * + * Return: pointer to erase type on success, NULL otherwise. + */ +static const struct spi_nor_erase_type * +spi_nor_select_uniform_erase(struct spi_nor_erase_map *map, + const u32 wanted_size) +{ + const struct spi_nor_erase_type *tested_erase, *erase = NULL; + int i; + u8 uniform_erase_type = map->uniform_erase_type; + + for (i = SNOR_ERASE_TYPE_MAX - 1; i >= 0; i--) { + if (!(uniform_erase_type & BIT(i))) + continue; + + tested_erase = &map->erase_type[i]; + + /* + * If the current erase size is the one, stop here: + * we have found the right uniform Sector Erase command. + */ + if (tested_erase->size == wanted_size) { + erase = tested_erase; + break; + } + + /* + * Otherwise, the current erase size is still a valid canditate. + * Select the biggest valid candidate. + */ + if (!erase && tested_erase->size) + erase = tested_erase; + /* keep iterating to find the wanted_size */ + } + + if (!erase) + return NULL; + + /* Disable all other Sector Erase commands. */ + map->uniform_erase_type &= ~SNOR_ERASE_TYPE_MASK; + map->uniform_erase_type |= BIT(erase - map->erase_type); + return erase; +} + +static int spi_nor_select_erase(struct spi_nor *nor) +{ + struct spi_nor_erase_map *map = &nor->params->erase_map; + const struct spi_nor_erase_type *erase = NULL; + struct mtd_info *mtd = &nor->mtd; + u32 wanted_size = nor->info->sector_size; + int i; + + /* + * The previous implementation handling Sector Erase commands assumed + * that the SPI flash memory has an uniform layout then used only one + * of the supported erase sizes for all Sector Erase commands. + * So to be backward compatible, the new implementation also tries to + * manage the SPI flash memory as uniform with a single erase sector + * size, when possible. + */ +#ifdef CONFIG_MTD_SPI_NOR_USE_4K_SECTORS + /* prefer "small sector" erase if possible */ + wanted_size = 4096u; +#endif + + if (spi_nor_has_uniform_erase(nor)) { + erase = spi_nor_select_uniform_erase(map, wanted_size); + if (!erase) + return -EINVAL; + nor->erase_opcode = erase->opcode; + mtd->erasesize = erase->size; + return 0; + } + + /* + * For non-uniform SPI flash memory, set mtd->erasesize to the + * maximum erase sector size. No need to set nor->erase_opcode. + */ + for (i = SNOR_ERASE_TYPE_MAX - 1; i >= 0; i--) { + if (map->erase_type[i].size) { + erase = &map->erase_type[i]; + break; + } + } + + if (!erase) + return -EINVAL; + + mtd->erasesize = erase->size; + return 0; +} + +static int spi_nor_default_setup(struct spi_nor *nor, + const struct spi_nor_hwcaps *hwcaps) +{ + struct spi_nor_flash_parameter *params = nor->params; + u32 ignored_mask, shared_mask; + int err; + + /* + * Keep only the hardware capabilities supported by both the SPI + * controller and the SPI flash memory. + */ + shared_mask = hwcaps->mask & params->hwcaps.mask; + + if (nor->spimem) { + /* + * When called from spi_nor_probe(), all caps are set and we + * need to discard some of them based on what the SPI + * controller actually supports (using spi_mem_supports_op()). + */ + spi_nor_spimem_adjust_hwcaps(nor, &shared_mask); + } else { + /* + * SPI n-n-n protocols are not supported when the SPI + * controller directly implements the spi_nor interface. + * Yet another reason to switch to spi-mem. + */ + ignored_mask = SNOR_HWCAPS_X_X_X; + if (shared_mask & ignored_mask) { + dev_dbg(nor->dev, + "SPI n-n-n protocols are not supported.\n"); + shared_mask &= ~ignored_mask; + } + } + + /* Select the (Fast) Read command. */ + err = spi_nor_select_read(nor, shared_mask); + if (err) { + dev_dbg(nor->dev, + "can't select read settings supported by both the SPI controller and memory.\n"); + return err; + } + + /* Select the Page Program command. */ + err = spi_nor_select_pp(nor, shared_mask); + if (err) { + dev_dbg(nor->dev, + "can't select write settings supported by both the SPI controller and memory.\n"); + return err; + } + + /* Select the Sector Erase command. */ + err = spi_nor_select_erase(nor); + if (err) { + dev_dbg(nor->dev, + "can't select erase settings supported by both the SPI controller and memory.\n"); + return err; + } + + return 0; +} + +static int spi_nor_setup(struct spi_nor *nor, + const struct spi_nor_hwcaps *hwcaps) +{ + if (!nor->params->setup) + return 0; + + return nor->params->setup(nor, hwcaps); +} + +/** + * spi_nor_manufacturer_init_params() - Initialize the flash's parameters and + * settings based on MFR register and ->default_init() hook. + * @nor: pointer to a 'struct spi-nor'. + */ +static void spi_nor_manufacturer_init_params(struct spi_nor *nor) +{ + if (nor->manufacturer && nor->manufacturer->fixups && + nor->manufacturer->fixups->default_init) + nor->manufacturer->fixups->default_init(nor); + + if (nor->info->fixups && nor->info->fixups->default_init) + nor->info->fixups->default_init(nor); +} + +/** + * spi_nor_sfdp_init_params() - Initialize the flash's parameters and settings + * based on JESD216 SFDP standard. + * @nor: pointer to a 'struct spi-nor'. + * + * The method has a roll-back mechanism: in case the SFDP parsing fails, the + * legacy flash parameters and settings will be restored. + */ +static void spi_nor_sfdp_init_params(struct spi_nor *nor) +{ + struct spi_nor_flash_parameter sfdp_params; + + memcpy(&sfdp_params, nor->params, sizeof(sfdp_params)); + + if (spi_nor_parse_sfdp(nor, &sfdp_params)) { + nor->addr_width = 0; + nor->flags &= ~SNOR_F_4B_OPCODES; + } else { + memcpy(nor->params, &sfdp_params, sizeof(*nor->params)); + } +} + +/** + * spi_nor_info_init_params() - Initialize the flash's parameters and settings + * based on nor->info data. + * @nor: pointer to a 'struct spi-nor'. + */ +static void spi_nor_info_init_params(struct spi_nor *nor) +{ + struct spi_nor_flash_parameter *params = nor->params; + struct spi_nor_erase_map *map = ¶ms->erase_map; + const struct flash_info *info = nor->info; + struct device_node *np = spi_nor_get_flash_node(nor); + u8 i, erase_mask; + + /* Initialize legacy flash parameters and settings. */ + params->quad_enable = spi_nor_sr2_bit1_quad_enable; + params->set_4byte_addr_mode = spansion_set_4byte_addr_mode; + params->setup = spi_nor_default_setup; + /* Default to 16-bit Write Status (01h) Command */ + nor->flags |= SNOR_F_HAS_16BIT_SR; + + /* Set SPI NOR sizes. */ + params->size = (u64)info->sector_size * info->n_sectors; + params->page_size = info->page_size; + + if (!(info->flags & SPI_NOR_NO_FR)) { + /* Default to Fast Read for DT and non-DT platform devices. */ + params->hwcaps.mask |= SNOR_HWCAPS_READ_FAST; + + /* Mask out Fast Read if not requested at DT instantiation. */ + if (np && !of_property_read_bool(np, "m25p,fast-read")) + params->hwcaps.mask &= ~SNOR_HWCAPS_READ_FAST; + } + + /* (Fast) Read settings. */ + params->hwcaps.mask |= SNOR_HWCAPS_READ; + spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ], + 0, 0, SPINOR_OP_READ, + SNOR_PROTO_1_1_1); + + if (params->hwcaps.mask & SNOR_HWCAPS_READ_FAST) + spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ_FAST], + 0, 8, SPINOR_OP_READ_FAST, + SNOR_PROTO_1_1_1); + + if (info->flags & SPI_NOR_DUAL_READ) { + params->hwcaps.mask |= SNOR_HWCAPS_READ_1_1_2; + spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ_1_1_2], + 0, 8, SPINOR_OP_READ_1_1_2, + SNOR_PROTO_1_1_2); + } + + if (info->flags & SPI_NOR_QUAD_READ) { + params->hwcaps.mask |= SNOR_HWCAPS_READ_1_1_4; + spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ_1_1_4], + 0, 8, SPINOR_OP_READ_1_1_4, + SNOR_PROTO_1_1_4); + } + + if (info->flags & SPI_NOR_OCTAL_READ) { + params->hwcaps.mask |= SNOR_HWCAPS_READ_1_1_8; + spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ_1_1_8], + 0, 8, SPINOR_OP_READ_1_1_8, + SNOR_PROTO_1_1_8); + } + + /* Page Program settings. */ + params->hwcaps.mask |= SNOR_HWCAPS_PP; + spi_nor_set_pp_settings(¶ms->page_programs[SNOR_CMD_PP], + SPINOR_OP_PP, SNOR_PROTO_1_1_1); + + /* + * Sector Erase settings. Sort Erase Types in ascending order, with the + * smallest erase size starting at BIT(0). + */ + erase_mask = 0; + i = 0; + if (info->flags & SECT_4K_PMC) { + erase_mask |= BIT(i); + spi_nor_set_erase_type(&map->erase_type[i], 4096u, + SPINOR_OP_BE_4K_PMC); + i++; + } else if (info->flags & SECT_4K) { + erase_mask |= BIT(i); + spi_nor_set_erase_type(&map->erase_type[i], 4096u, + SPINOR_OP_BE_4K); + i++; + } + erase_mask |= BIT(i); + spi_nor_set_erase_type(&map->erase_type[i], info->sector_size, + SPINOR_OP_SE); + spi_nor_init_uniform_erase_map(map, erase_mask, params->size); +} + +/** + * spi_nor_post_sfdp_fixups() - Updates the flash's parameters and settings + * after SFDP has been parsed (is also called for SPI NORs that do not + * support RDSFDP). + * @nor: pointer to a 'struct spi_nor' + * + * Typically used to tweak various parameters that could not be extracted by + * other means (i.e. when information provided by the SFDP/flash_info tables + * are incomplete or wrong). + */ +static void spi_nor_post_sfdp_fixups(struct spi_nor *nor) +{ + if (nor->manufacturer && nor->manufacturer->fixups && + nor->manufacturer->fixups->post_sfdp) + nor->manufacturer->fixups->post_sfdp(nor); + + if (nor->info->fixups && nor->info->fixups->post_sfdp) + nor->info->fixups->post_sfdp(nor); +} + +/** + * spi_nor_late_init_params() - Late initialization of default flash parameters. + * @nor: pointer to a 'struct spi_nor' + * + * Used to set default flash parameters and settings when the ->default_init() + * hook or the SFDP parser let voids. + */ +static void spi_nor_late_init_params(struct spi_nor *nor) +{ + /* + * NOR protection support. When locking_ops are not provided, we pick + * the default ones. + */ + if (nor->flags & SNOR_F_HAS_LOCK && !nor->params->locking_ops) + nor->params->locking_ops = &spi_nor_sr_locking_ops; +} + +/** + * spi_nor_init_params() - Initialize the flash's parameters and settings. + * @nor: pointer to a 'struct spi-nor'. + * + * The flash parameters and settings are initialized based on a sequence of + * calls that are ordered by priority: + * + * 1/ Default flash parameters initialization. The initializations are done + * based on nor->info data: + * spi_nor_info_init_params() + * + * which can be overwritten by: + * 2/ Manufacturer flash parameters initialization. The initializations are + * done based on MFR register, or when the decisions can not be done solely + * based on MFR, by using specific flash_info tweeks, ->default_init(): + * spi_nor_manufacturer_init_params() + * + * which can be overwritten by: + * 3/ SFDP flash parameters initialization. JESD216 SFDP is a standard and + * should be more accurate that the above. + * spi_nor_sfdp_init_params() + * + * Please note that there is a ->post_bfpt() fixup hook that can overwrite + * the flash parameters and settings immediately after parsing the Basic + * Flash Parameter Table. + * + * which can be overwritten by: + * 4/ Post SFDP flash parameters initialization. Used to tweak various + * parameters that could not be extracted by other means (i.e. when + * information provided by the SFDP/flash_info tables are incomplete or + * wrong). + * spi_nor_post_sfdp_fixups() + * + * 5/ Late default flash parameters initialization, used when the + * ->default_init() hook or the SFDP parser do not set specific params. + * spi_nor_late_init_params() + */ +static int spi_nor_init_params(struct spi_nor *nor) +{ + nor->params = devm_kzalloc(nor->dev, sizeof(*nor->params), GFP_KERNEL); + if (!nor->params) + return -ENOMEM; + + spi_nor_info_init_params(nor); + + spi_nor_manufacturer_init_params(nor); + + if ((nor->info->flags & (SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ)) && + !(nor->info->flags & SPI_NOR_SKIP_SFDP)) + spi_nor_sfdp_init_params(nor); + + spi_nor_post_sfdp_fixups(nor); + + spi_nor_late_init_params(nor); + + return 0; +} + +/** + * spi_nor_quad_enable() - enable Quad I/O if needed. + * @nor: pointer to a 'struct spi_nor' + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_quad_enable(struct spi_nor *nor) +{ + if (!nor->params->quad_enable) + return 0; + + if (!(spi_nor_get_protocol_width(nor->read_proto) == 4 || + spi_nor_get_protocol_width(nor->write_proto) == 4)) + return 0; + + return nor->params->quad_enable(nor); +} + +/** + * spi_nor_unlock_all() - Unlocks the entire flash memory array. + * @nor: pointer to a 'struct spi_nor'. + * + * Some SPI NOR flashes are write protected by default after a power-on reset + * cycle, in order to avoid inadvertent writes during power-up. Backward + * compatibility imposes to unlock the entire flash memory array at power-up + * by default. + */ +static int spi_nor_unlock_all(struct spi_nor *nor) +{ + if (nor->flags & SNOR_F_HAS_LOCK) + return spi_nor_unlock(&nor->mtd, 0, nor->params->size); + + return 0; +} + +static int spi_nor_init(struct spi_nor *nor) +{ + int err; + + err = spi_nor_quad_enable(nor); + if (err) { + dev_dbg(nor->dev, "quad mode not supported\n"); + return err; + } + + err = spi_nor_unlock_all(nor); + if (err) { + dev_dbg(nor->dev, "Failed to unlock the entire flash memory array\n"); + return err; + } + + if (nor->addr_width == 4 && !(nor->flags & SNOR_F_4B_OPCODES)) { + /* + * If the RESET# pin isn't hooked up properly, or the system + * otherwise doesn't perform a reset command in the boot + * sequence, it's impossible to 100% protect against unexpected + * reboots (e.g., crashes). Warn the user (or hopefully, system + * designer) that this is bad. + */ + WARN_ONCE(nor->flags & SNOR_F_BROKEN_RESET, + "enabling reset hack; may not recover from unexpected reboots\n"); + nor->params->set_4byte_addr_mode(nor, true); + } + + return 0; +} + +/* mtd resume handler */ +static void spi_nor_resume(struct mtd_info *mtd) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + struct device *dev = nor->dev; + int ret; + + /* re-initialize the nor chip */ + ret = spi_nor_init(nor); + if (ret) + dev_err(dev, "resume() failed\n"); +} + +void spi_nor_restore(struct spi_nor *nor) +{ + /* restore the addressing mode */ + if (nor->addr_width == 4 && !(nor->flags & SNOR_F_4B_OPCODES) && + nor->flags & SNOR_F_BROKEN_RESET) + nor->params->set_4byte_addr_mode(nor, false); +} +EXPORT_SYMBOL_GPL(spi_nor_restore); + +static const struct flash_info *spi_nor_match_id(struct spi_nor *nor, + const char *name) +{ + unsigned int i, j; + + for (i = 0; i < ARRAY_SIZE(manufacturers); i++) { + for (j = 0; j < manufacturers[i]->nparts; j++) { + if (!strcmp(name, manufacturers[i]->parts[j].name)) { + nor->manufacturer = manufacturers[i]; + return &manufacturers[i]->parts[j]; + } + } + } + + return NULL; +} + +static int spi_nor_set_addr_width(struct spi_nor *nor) +{ + if (nor->addr_width) { + /* already configured from SFDP */ + } else if (nor->info->addr_width) { + nor->addr_width = nor->info->addr_width; + } else if (nor->mtd.size > 0x1000000) { + /* enable 4-byte addressing if the device exceeds 16MiB */ + nor->addr_width = 4; + } else { + nor->addr_width = 3; + } + + if (nor->addr_width > SPI_NOR_MAX_ADDR_WIDTH) { + dev_dbg(nor->dev, "address width is too large: %u\n", + nor->addr_width); + return -EINVAL; + } + + /* Set 4byte opcodes when possible. */ + if (nor->addr_width == 4 && nor->flags & SNOR_F_4B_OPCODES && + !(nor->flags & SNOR_F_HAS_4BAIT)) + spi_nor_set_4byte_opcodes(nor); + + return 0; +} + +static void spi_nor_debugfs_init(struct spi_nor *nor, + const struct flash_info *info) +{ + struct mtd_info *mtd = &nor->mtd; + + mtd->dbg.partname = info->name; + mtd->dbg.partid = devm_kasprintf(nor->dev, GFP_KERNEL, "spi-nor:%*phN", + info->id_len, info->id); +} + +static const struct flash_info *spi_nor_get_flash_info(struct spi_nor *nor, + const char *name) +{ + const struct flash_info *info = NULL; + + if (name) + info = spi_nor_match_id(nor, name); + /* Try to auto-detect if chip name wasn't specified or not found */ + if (!info) + info = spi_nor_read_id(nor); + if (IS_ERR_OR_NULL(info)) + return ERR_PTR(-ENOENT); + + /* + * If caller has specified name of flash model that can normally be + * detected using JEDEC, let's verify it. + */ + if (name && info->id_len) { + const struct flash_info *jinfo; + + jinfo = spi_nor_read_id(nor); + if (IS_ERR(jinfo)) { + return jinfo; + } else if (jinfo != info) { + /* + * JEDEC knows better, so overwrite platform ID. We + * can't trust partitions any longer, but we'll let + * mtd apply them anyway, since some partitions may be + * marked read-only, and we don't want to lose that + * information, even if it's not 100% accurate. + */ + dev_warn(nor->dev, "found %s, expected %s\n", + jinfo->name, info->name); + info = jinfo; + } + } + + return info; +} + +int spi_nor_scan(struct spi_nor *nor, const char *name, + const struct spi_nor_hwcaps *hwcaps) +{ + const struct flash_info *info; + struct device *dev = nor->dev; + struct mtd_info *mtd = &nor->mtd; + struct device_node *np = spi_nor_get_flash_node(nor); + int ret; + int i; + + ret = spi_nor_check(nor); + if (ret) + return ret; + + /* Reset SPI protocol for all commands. */ + nor->reg_proto = SNOR_PROTO_1_1_1; + nor->read_proto = SNOR_PROTO_1_1_1; + nor->write_proto = SNOR_PROTO_1_1_1; + + /* + * We need the bounce buffer early to read/write registers when going + * through the spi-mem layer (buffers have to be DMA-able). + * For spi-mem drivers, we'll reallocate a new buffer if + * nor->page_size turns out to be greater than PAGE_SIZE (which + * shouldn't happen before long since NOR pages are usually less + * than 1KB) after spi_nor_scan() returns. + */ + nor->bouncebuf_size = PAGE_SIZE; + nor->bouncebuf = devm_kmalloc(dev, nor->bouncebuf_size, + GFP_KERNEL); + if (!nor->bouncebuf) + return -ENOMEM; + + info = spi_nor_get_flash_info(nor, name); + if (IS_ERR(info)) + return PTR_ERR(info); + + nor->info = info; + + spi_nor_debugfs_init(nor, info); + + mutex_init(&nor->lock); + + /* + * Make sure the XSR_RDY flag is set before calling + * spi_nor_wait_till_ready(). Xilinx S3AN share MFR + * with Atmel spi-nor + */ + if (info->flags & SPI_NOR_XSR_RDY) + nor->flags |= SNOR_F_READY_XSR_RDY; + + if (info->flags & SPI_NOR_HAS_LOCK) + nor->flags |= SNOR_F_HAS_LOCK; + + mtd->_write = spi_nor_write; + + /* Init flash parameters based on flash_info struct and SFDP */ + ret = spi_nor_init_params(nor); + if (ret) + return ret; + + if (!mtd->name) + mtd->name = dev_name(dev); + mtd->priv = nor; + mtd->type = MTD_NORFLASH; + mtd->writesize = 1; + mtd->flags = MTD_CAP_NORFLASH; + mtd->size = nor->params->size; + mtd->_erase = spi_nor_erase; + mtd->_read = spi_nor_read; + mtd->_resume = spi_nor_resume; + + if (nor->params->locking_ops) { + mtd->_lock = spi_nor_lock; + mtd->_unlock = spi_nor_unlock; + mtd->_is_locked = spi_nor_is_locked; + } + + if (info->flags & USE_FSR) + nor->flags |= SNOR_F_USE_FSR; + if (info->flags & SPI_NOR_HAS_TB) { + nor->flags |= SNOR_F_HAS_SR_TB; + if (info->flags & SPI_NOR_TB_SR_BIT6) + nor->flags |= SNOR_F_HAS_SR_TB_BIT6; + } + + if (info->flags & NO_CHIP_ERASE) + nor->flags |= SNOR_F_NO_OP_CHIP_ERASE; + if (info->flags & USE_CLSR) + nor->flags |= SNOR_F_USE_CLSR; + + if (info->flags & SPI_NOR_4BIT_BP) { + nor->flags |= SNOR_F_HAS_4BIT_BP; + if (info->flags & SPI_NOR_BP3_SR_BIT6) + nor->flags |= SNOR_F_HAS_SR_BP3_BIT6; + } + + if (info->flags & SPI_NOR_NO_ERASE) + mtd->flags |= MTD_NO_ERASE; + + mtd->dev.parent = dev; + nor->page_size = nor->params->page_size; + mtd->writebufsize = nor->page_size; + + if (of_property_read_bool(np, "broken-flash-reset")) + nor->flags |= SNOR_F_BROKEN_RESET; + + /* + * Configure the SPI memory: + * - select op codes for (Fast) Read, Page Program and Sector Erase. + * - set the number of dummy cycles (mode cycles + wait states). + * - set the SPI protocols for register and memory accesses. + */ + ret = spi_nor_setup(nor, hwcaps); + if (ret) + return ret; + + if (info->flags & SPI_NOR_4B_OPCODES) + nor->flags |= SNOR_F_4B_OPCODES; + + ret = spi_nor_set_addr_width(nor); + if (ret) + return ret; + + /* Send all the required SPI flash commands to initialize device */ + ret = spi_nor_init(nor); + if (ret) + return ret; + + dev_info(dev, "%s (%lld Kbytes)\n", info->name, + (long long)mtd->size >> 10); + + dev_dbg(dev, + "mtd .name = %s, .size = 0x%llx (%lldMiB), " + ".erasesize = 0x%.8x (%uKiB) .numeraseregions = %d\n", + mtd->name, (long long)mtd->size, (long long)(mtd->size >> 20), + mtd->erasesize, mtd->erasesize / 1024, mtd->numeraseregions); + + if (mtd->numeraseregions) + for (i = 0; i < mtd->numeraseregions; i++) + dev_dbg(dev, + "mtd.eraseregions[%d] = { .offset = 0x%llx, " + ".erasesize = 0x%.8x (%uKiB), " + ".numblocks = %d }\n", + i, (long long)mtd->eraseregions[i].offset, + mtd->eraseregions[i].erasesize, + mtd->eraseregions[i].erasesize / 1024, + mtd->eraseregions[i].numblocks); + return 0; +} +EXPORT_SYMBOL_GPL(spi_nor_scan); + +static int spi_nor_create_read_dirmap(struct spi_nor *nor) +{ + struct spi_mem_dirmap_info info = { + .op_tmpl = SPI_MEM_OP(SPI_MEM_OP_CMD(nor->read_opcode, 1), + SPI_MEM_OP_ADDR(nor->addr_width, 0, 1), + SPI_MEM_OP_DUMMY(nor->read_dummy, 1), + SPI_MEM_OP_DATA_IN(0, NULL, 1)), + .offset = 0, + .length = nor->mtd.size, + }; + struct spi_mem_op *op = &info.op_tmpl; + + /* get transfer protocols. */ + op->cmd.buswidth = spi_nor_get_protocol_inst_nbits(nor->read_proto); + op->addr.buswidth = spi_nor_get_protocol_addr_nbits(nor->read_proto); + op->dummy.buswidth = op->addr.buswidth; + op->data.buswidth = spi_nor_get_protocol_data_nbits(nor->read_proto); + + /* convert the dummy cycles to the number of bytes */ + op->dummy.nbytes = (nor->read_dummy * op->dummy.buswidth) / 8; + + nor->dirmap.rdesc = devm_spi_mem_dirmap_create(nor->dev, nor->spimem, + &info); + return PTR_ERR_OR_ZERO(nor->dirmap.rdesc); +} + +static int spi_nor_create_write_dirmap(struct spi_nor *nor) +{ + struct spi_mem_dirmap_info info = { + .op_tmpl = SPI_MEM_OP(SPI_MEM_OP_CMD(nor->program_opcode, 1), + SPI_MEM_OP_ADDR(nor->addr_width, 0, 1), + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_OUT(0, NULL, 1)), + .offset = 0, + .length = nor->mtd.size, + }; + struct spi_mem_op *op = &info.op_tmpl; + + /* get transfer protocols. */ + op->cmd.buswidth = spi_nor_get_protocol_inst_nbits(nor->write_proto); + op->addr.buswidth = spi_nor_get_protocol_addr_nbits(nor->write_proto); + op->dummy.buswidth = op->addr.buswidth; + op->data.buswidth = spi_nor_get_protocol_data_nbits(nor->write_proto); + + if (nor->program_opcode == SPINOR_OP_AAI_WP && nor->sst_write_second) + op->addr.nbytes = 0; + + nor->dirmap.wdesc = devm_spi_mem_dirmap_create(nor->dev, nor->spimem, + &info); + return PTR_ERR_OR_ZERO(nor->dirmap.wdesc); +} + +static int spi_nor_probe(struct spi_mem *spimem) +{ + struct spi_device *spi = spimem->spi; + struct flash_platform_data *data = dev_get_platdata(&spi->dev); + struct spi_nor *nor; + /* + * Enable all caps by default. The core will mask them after + * checking what's really supported using spi_mem_supports_op(). + */ + const struct spi_nor_hwcaps hwcaps = { .mask = SNOR_HWCAPS_ALL }; + char *flash_name; + int ret; + + nor = devm_kzalloc(&spi->dev, sizeof(*nor), GFP_KERNEL); + if (!nor) + return -ENOMEM; + + nor->spimem = spimem; + nor->dev = &spi->dev; + spi_nor_set_flash_node(nor, spi->dev.of_node); + + spi_mem_set_drvdata(spimem, nor); + + if (data && data->name) + nor->mtd.name = data->name; + + if (!nor->mtd.name) + nor->mtd.name = spi_mem_get_name(spimem); + + /* + * For some (historical?) reason many platforms provide two different + * names in flash_platform_data: "name" and "type". Quite often name is + * set to "m25p80" and then "type" provides a real chip name. + * If that's the case, respect "type" and ignore a "name". + */ + if (data && data->type) + flash_name = data->type; + else if (!strcmp(spi->modalias, "spi-nor")) + flash_name = NULL; /* auto-detect */ + else + flash_name = spi->modalias; + + ret = spi_nor_scan(nor, flash_name, &hwcaps); + if (ret) + return ret; + + /* + * None of the existing parts have > 512B pages, but let's play safe + * and add this logic so that if anyone ever adds support for such + * a NOR we don't end up with buffer overflows. + */ + if (nor->page_size > PAGE_SIZE) { + nor->bouncebuf_size = nor->page_size; + devm_kfree(nor->dev, nor->bouncebuf); + nor->bouncebuf = devm_kmalloc(nor->dev, + nor->bouncebuf_size, + GFP_KERNEL); + if (!nor->bouncebuf) + return -ENOMEM; + } + + ret = spi_nor_create_read_dirmap(nor); + if (ret) + return ret; + + ret = spi_nor_create_write_dirmap(nor); + if (ret) + return ret; + + return mtd_device_register(&nor->mtd, data ? data->parts : NULL, + data ? data->nr_parts : 0); +} + +static int spi_nor_remove(struct spi_mem *spimem) +{ + struct spi_nor *nor = spi_mem_get_drvdata(spimem); + + spi_nor_restore(nor); + + /* Clean up MTD stuff. */ + return mtd_device_unregister(&nor->mtd); +} + +static void spi_nor_shutdown(struct spi_mem *spimem) +{ + struct spi_nor *nor = spi_mem_get_drvdata(spimem); + + spi_nor_restore(nor); +} + +/* + * Do NOT add to this array without reading the following: + * + * Historically, many flash devices are bound to this driver by their name. But + * since most of these flash are compatible to some extent, and their + * differences can often be differentiated by the JEDEC read-ID command, we + * encourage new users to add support to the spi-nor library, and simply bind + * against a generic string here (e.g., "jedec,spi-nor"). + * + * Many flash names are kept here in this list (as well as in spi-nor.c) to + * keep them available as module aliases for existing platforms. + */ +static const struct spi_device_id spi_nor_dev_ids[] = { + /* + * Allow non-DT platform devices to bind to the "spi-nor" modalias, and + * hack around the fact that the SPI core does not provide uevent + * matching for .of_match_table + */ + {"spi-nor"}, + + /* + * Entries not used in DTs that should be safe to drop after replacing + * them with "spi-nor" in platform data. + */ + {"s25sl064a"}, {"w25x16"}, {"m25p10"}, {"m25px64"}, + + /* + * Entries that were used in DTs without "jedec,spi-nor" fallback and + * should be kept for backward compatibility. + */ + {"at25df321a"}, {"at25df641"}, {"at26df081a"}, + {"mx25l4005a"}, {"mx25l1606e"}, {"mx25l6405d"}, {"mx25l12805d"}, + {"mx25l25635e"},{"mx66l51235l"}, + {"n25q064"}, {"n25q128a11"}, {"n25q128a13"}, {"n25q512a"}, + {"s25fl256s1"}, {"s25fl512s"}, {"s25sl12801"}, {"s25fl008k"}, + {"s25fl064k"}, + {"sst25vf040b"},{"sst25vf016b"},{"sst25vf032b"},{"sst25wf040"}, + {"m25p40"}, {"m25p80"}, {"m25p16"}, {"m25p32"}, + {"m25p64"}, {"m25p128"}, + {"w25x80"}, {"w25x32"}, {"w25q32"}, {"w25q32dw"}, + {"w25q80bl"}, {"w25q128"}, {"w25q256"}, + + /* Flashes that can't be detected using JEDEC */ + {"m25p05-nonjedec"}, {"m25p10-nonjedec"}, {"m25p20-nonjedec"}, + {"m25p40-nonjedec"}, {"m25p80-nonjedec"}, {"m25p16-nonjedec"}, + {"m25p32-nonjedec"}, {"m25p64-nonjedec"}, {"m25p128-nonjedec"}, + + /* Everspin MRAMs (non-JEDEC) */ + { "mr25h128" }, /* 128 Kib, 40 MHz */ + { "mr25h256" }, /* 256 Kib, 40 MHz */ + { "mr25h10" }, /* 1 Mib, 40 MHz */ + { "mr25h40" }, /* 4 Mib, 40 MHz */ + + { }, +}; +MODULE_DEVICE_TABLE(spi, spi_nor_dev_ids); + +static const struct of_device_id spi_nor_of_table[] = { + /* + * Generic compatibility for SPI NOR that can be identified by the + * JEDEC READ ID opcode (0x9F). Use this, if possible. + */ + { .compatible = "jedec,spi-nor" }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, spi_nor_of_table); + +/* + * REVISIT: many of these chips have deep power-down modes, which + * should clearly be entered on suspend() to minimize power use. + * And also when they're otherwise idle... + */ +static struct spi_mem_driver spi_nor_driver = { + .spidrv = { + .driver = { + .name = "spi-nor", + .of_match_table = spi_nor_of_table, + }, + .id_table = spi_nor_dev_ids, + }, + .probe = spi_nor_probe, + .remove = spi_nor_remove, + .shutdown = spi_nor_shutdown, +}; +module_spi_mem_driver(spi_nor_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Huang Shijie "); +MODULE_AUTHOR("Mike Lavender"); +MODULE_DESCRIPTION("framework for SPI NOR"); diff --git a/drivers/mtd/spi-nor/core.h b/drivers/mtd/spi-nor/core.h new file mode 100644 index 000000000000..6f2f6b27173f --- /dev/null +++ b/drivers/mtd/spi-nor/core.h @@ -0,0 +1,441 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#ifndef __LINUX_MTD_SPI_NOR_INTERNAL_H +#define __LINUX_MTD_SPI_NOR_INTERNAL_H + +#include "sfdp.h" + +#define SPI_NOR_MAX_ID_LEN 6 + +enum spi_nor_option_flags { + SNOR_F_USE_FSR = BIT(0), + SNOR_F_HAS_SR_TB = BIT(1), + SNOR_F_NO_OP_CHIP_ERASE = BIT(2), + SNOR_F_READY_XSR_RDY = BIT(3), + SNOR_F_USE_CLSR = BIT(4), + SNOR_F_BROKEN_RESET = BIT(5), + SNOR_F_4B_OPCODES = BIT(6), + SNOR_F_HAS_4BAIT = BIT(7), + SNOR_F_HAS_LOCK = BIT(8), + SNOR_F_HAS_16BIT_SR = BIT(9), + SNOR_F_NO_READ_CR = BIT(10), + SNOR_F_HAS_SR_TB_BIT6 = BIT(11), + SNOR_F_HAS_4BIT_BP = BIT(12), + SNOR_F_HAS_SR_BP3_BIT6 = BIT(13), +}; + +struct spi_nor_read_command { + u8 num_mode_clocks; + u8 num_wait_states; + u8 opcode; + enum spi_nor_protocol proto; +}; + +struct spi_nor_pp_command { + u8 opcode; + enum spi_nor_protocol proto; +}; + +enum spi_nor_read_command_index { + SNOR_CMD_READ, + SNOR_CMD_READ_FAST, + SNOR_CMD_READ_1_1_1_DTR, + + /* Dual SPI */ + SNOR_CMD_READ_1_1_2, + SNOR_CMD_READ_1_2_2, + SNOR_CMD_READ_2_2_2, + SNOR_CMD_READ_1_2_2_DTR, + + /* Quad SPI */ + SNOR_CMD_READ_1_1_4, + SNOR_CMD_READ_1_4_4, + SNOR_CMD_READ_4_4_4, + SNOR_CMD_READ_1_4_4_DTR, + + /* Octal SPI */ + SNOR_CMD_READ_1_1_8, + SNOR_CMD_READ_1_8_8, + SNOR_CMD_READ_8_8_8, + SNOR_CMD_READ_1_8_8_DTR, + + SNOR_CMD_READ_MAX +}; + +enum spi_nor_pp_command_index { + SNOR_CMD_PP, + + /* Quad SPI */ + SNOR_CMD_PP_1_1_4, + SNOR_CMD_PP_1_4_4, + SNOR_CMD_PP_4_4_4, + + /* Octal SPI */ + SNOR_CMD_PP_1_1_8, + SNOR_CMD_PP_1_8_8, + SNOR_CMD_PP_8_8_8, + + SNOR_CMD_PP_MAX +}; + +/** + * struct spi_nor_erase_type - Structure to describe a SPI NOR erase type + * @size: the size of the sector/block erased by the erase type. + * JEDEC JESD216B imposes erase sizes to be a power of 2. + * @size_shift: @size is a power of 2, the shift is stored in + * @size_shift. + * @size_mask: the size mask based on @size_shift. + * @opcode: the SPI command op code to erase the sector/block. + * @idx: Erase Type index as sorted in the Basic Flash Parameter + * Table. It will be used to synchronize the supported + * Erase Types with the ones identified in the SFDP + * optional tables. + */ +struct spi_nor_erase_type { + u32 size; + u32 size_shift; + u32 size_mask; + u8 opcode; + u8 idx; +}; + +/** + * struct spi_nor_erase_command - Used for non-uniform erases + * The structure is used to describe a list of erase commands to be executed + * once we validate that the erase can be performed. The elements in the list + * are run-length encoded. + * @list: for inclusion into the list of erase commands. + * @count: how many times the same erase command should be + * consecutively used. + * @size: the size of the sector/block erased by the command. + * @opcode: the SPI command op code to erase the sector/block. + */ +struct spi_nor_erase_command { + struct list_head list; + u32 count; + u32 size; + u8 opcode; +}; + +/** + * struct spi_nor_erase_region - Structure to describe a SPI NOR erase region + * @offset: the offset in the data array of erase region start. + * LSB bits are used as a bitmask encoding flags to + * determine if this region is overlaid, if this region is + * the last in the SPI NOR flash memory and to indicate + * all the supported erase commands inside this region. + * The erase types are sorted in ascending order with the + * smallest Erase Type size being at BIT(0). + * @size: the size of the region in bytes. + */ +struct spi_nor_erase_region { + u64 offset; + u64 size; +}; + +#define SNOR_ERASE_TYPE_MAX 4 +#define SNOR_ERASE_TYPE_MASK GENMASK_ULL(SNOR_ERASE_TYPE_MAX - 1, 0) + +#define SNOR_LAST_REGION BIT(4) +#define SNOR_OVERLAID_REGION BIT(5) + +#define SNOR_ERASE_FLAGS_MAX 6 +#define SNOR_ERASE_FLAGS_MASK GENMASK_ULL(SNOR_ERASE_FLAGS_MAX - 1, 0) + +/** + * struct spi_nor_erase_map - Structure to describe the SPI NOR erase map + * @regions: array of erase regions. The regions are consecutive in + * address space. Walking through the regions is done + * incrementally. + * @uniform_region: a pre-allocated erase region for SPI NOR with a uniform + * sector size (legacy implementation). + * @erase_type: an array of erase types shared by all the regions. + * The erase types are sorted in ascending order, with the + * smallest Erase Type size being the first member in the + * erase_type array. + * @uniform_erase_type: bitmask encoding erase types that can erase the + * entire memory. This member is completed at init by + * uniform and non-uniform SPI NOR flash memories if they + * support at least one erase type that can erase the + * entire memory. + */ +struct spi_nor_erase_map { + struct spi_nor_erase_region *regions; + struct spi_nor_erase_region uniform_region; + struct spi_nor_erase_type erase_type[SNOR_ERASE_TYPE_MAX]; + u8 uniform_erase_type; +}; + +/** + * struct spi_nor_locking_ops - SPI NOR locking methods + * @lock: lock a region of the SPI NOR. + * @unlock: unlock a region of the SPI NOR. + * @is_locked: check if a region of the SPI NOR is completely locked + */ +struct spi_nor_locking_ops { + int (*lock)(struct spi_nor *nor, loff_t ofs, uint64_t len); + int (*unlock)(struct spi_nor *nor, loff_t ofs, uint64_t len); + int (*is_locked)(struct spi_nor *nor, loff_t ofs, uint64_t len); +}; + +/** + * struct spi_nor_flash_parameter - SPI NOR flash parameters and settings. + * Includes legacy flash parameters and settings that can be overwritten + * by the spi_nor_fixups hooks, or dynamically when parsing the JESD216 + * Serial Flash Discoverable Parameters (SFDP) tables. + * + * @size: the flash memory density in bytes. + * @page_size: the page size of the SPI NOR flash memory. + * @hwcaps: describes the read and page program hardware + * capabilities. + * @reads: read capabilities ordered by priority: the higher index + * in the array, the higher priority. + * @page_programs: page program capabilities ordered by priority: the + * higher index in the array, the higher priority. + * @erase_map: the erase map parsed from the SFDP Sector Map Parameter + * Table. + * @quad_enable: enables SPI NOR quad mode. + * @set_4byte_addr_mode: puts the SPI NOR in 4 byte addressing mode. + * @convert_addr: converts an absolute address into something the flash + * will understand. Particularly useful when pagesize is + * not a power-of-2. + * @setup: configures the SPI NOR memory. Useful for SPI NOR + * flashes that have peculiarities to the SPI NOR standard + * e.g. different opcodes, specific address calculation, + * page size, etc. + * @locking_ops: SPI NOR locking methods. + */ +struct spi_nor_flash_parameter { + u64 size; + u32 page_size; + + struct spi_nor_hwcaps hwcaps; + struct spi_nor_read_command reads[SNOR_CMD_READ_MAX]; + struct spi_nor_pp_command page_programs[SNOR_CMD_PP_MAX]; + + struct spi_nor_erase_map erase_map; + + int (*quad_enable)(struct spi_nor *nor); + int (*set_4byte_addr_mode)(struct spi_nor *nor, bool enable); + u32 (*convert_addr)(struct spi_nor *nor, u32 addr); + int (*setup)(struct spi_nor *nor, const struct spi_nor_hwcaps *hwcaps); + + const struct spi_nor_locking_ops *locking_ops; +}; + +/** + * struct spi_nor_fixups - SPI NOR fixup hooks + * @default_init: called after default flash parameters init. Used to tweak + * flash parameters when information provided by the flash_info + * table is incomplete or wrong. + * @post_bfpt: called after the BFPT table has been parsed + * @post_sfdp: called after SFDP has been parsed (is also called for SPI NORs + * that do not support RDSFDP). Typically used to tweak various + * parameters that could not be extracted by other means (i.e. + * when information provided by the SFDP/flash_info tables are + * incomplete or wrong). + * + * Those hooks can be used to tweak the SPI NOR configuration when the SFDP + * table is broken or not available. + */ +struct spi_nor_fixups { + void (*default_init)(struct spi_nor *nor); + int (*post_bfpt)(struct spi_nor *nor, + const struct sfdp_parameter_header *bfpt_header, + const struct sfdp_bfpt *bfpt, + struct spi_nor_flash_parameter *params); + void (*post_sfdp)(struct spi_nor *nor); +}; + +struct flash_info { + char *name; + + /* + * This array stores the ID bytes. + * The first three bytes are the JEDIC ID. + * JEDEC ID zero means "no ID" (mostly older chips). + */ + u8 id[SPI_NOR_MAX_ID_LEN]; + u8 id_len; + + /* The size listed here is what works with SPINOR_OP_SE, which isn't + * necessarily called a "sector" by the vendor. + */ + unsigned sector_size; + u16 n_sectors; + + u16 page_size; + u16 addr_width; + + u32 flags; +#define SECT_4K BIT(0) /* SPINOR_OP_BE_4K works uniformly */ +#define SPI_NOR_NO_ERASE BIT(1) /* No erase command needed */ +#define SST_WRITE BIT(2) /* use SST byte programming */ +#define SPI_NOR_NO_FR BIT(3) /* Can't do fastread */ +#define SECT_4K_PMC BIT(4) /* SPINOR_OP_BE_4K_PMC works uniformly */ +#define SPI_NOR_DUAL_READ BIT(5) /* Flash supports Dual Read */ +#define SPI_NOR_QUAD_READ BIT(6) /* Flash supports Quad Read */ +#define USE_FSR BIT(7) /* use flag status register */ +#define SPI_NOR_HAS_LOCK BIT(8) /* Flash supports lock/unlock via SR */ +#define SPI_NOR_HAS_TB BIT(9) /* + * Flash SR has Top/Bottom (TB) protect + * bit. Must be used with + * SPI_NOR_HAS_LOCK. + */ +#define SPI_NOR_XSR_RDY BIT(10) /* + * S3AN flashes have specific opcode to + * read the status register. + */ +#define SPI_NOR_4B_OPCODES BIT(11) /* + * Use dedicated 4byte address op codes + * to support memory size above 128Mib. + */ +#define NO_CHIP_ERASE BIT(12) /* Chip does not support chip erase */ +#define SPI_NOR_SKIP_SFDP BIT(13) /* Skip parsing of SFDP tables */ +#define USE_CLSR BIT(14) /* use CLSR command */ +#define SPI_NOR_OCTAL_READ BIT(15) /* Flash supports Octal Read */ +#define SPI_NOR_TB_SR_BIT6 BIT(16) /* + * Top/Bottom (TB) is bit 6 of + * status register. Must be used with + * SPI_NOR_HAS_TB. + */ +#define SPI_NOR_4BIT_BP BIT(17) /* + * Flash SR has 4 bit fields (BP0-3) + * for block protection. + */ +#define SPI_NOR_BP3_SR_BIT6 BIT(18) /* + * BP3 is bit 6 of status register. + * Must be used with SPI_NOR_4BIT_BP. + */ + + /* Part specific fixup hooks. */ + const struct spi_nor_fixups *fixups; +}; + +/* Used when the "_ext_id" is two bytes at most */ +#define INFO(_jedec_id, _ext_id, _sector_size, _n_sectors, _flags) \ + .id = { \ + ((_jedec_id) >> 16) & 0xff, \ + ((_jedec_id) >> 8) & 0xff, \ + (_jedec_id) & 0xff, \ + ((_ext_id) >> 8) & 0xff, \ + (_ext_id) & 0xff, \ + }, \ + .id_len = (!(_jedec_id) ? 0 : (3 + ((_ext_id) ? 2 : 0))), \ + .sector_size = (_sector_size), \ + .n_sectors = (_n_sectors), \ + .page_size = 256, \ + .flags = (_flags), + +#define INFO6(_jedec_id, _ext_id, _sector_size, _n_sectors, _flags) \ + .id = { \ + ((_jedec_id) >> 16) & 0xff, \ + ((_jedec_id) >> 8) & 0xff, \ + (_jedec_id) & 0xff, \ + ((_ext_id) >> 16) & 0xff, \ + ((_ext_id) >> 8) & 0xff, \ + (_ext_id) & 0xff, \ + }, \ + .id_len = 6, \ + .sector_size = (_sector_size), \ + .n_sectors = (_n_sectors), \ + .page_size = 256, \ + .flags = (_flags), + +#define CAT25_INFO(_sector_size, _n_sectors, _page_size, _addr_width, _flags) \ + .sector_size = (_sector_size), \ + .n_sectors = (_n_sectors), \ + .page_size = (_page_size), \ + .addr_width = (_addr_width), \ + .flags = (_flags), + +#define S3AN_INFO(_jedec_id, _n_sectors, _page_size) \ + .id = { \ + ((_jedec_id) >> 16) & 0xff, \ + ((_jedec_id) >> 8) & 0xff, \ + (_jedec_id) & 0xff \ + }, \ + .id_len = 3, \ + .sector_size = (8*_page_size), \ + .n_sectors = (_n_sectors), \ + .page_size = _page_size, \ + .addr_width = 3, \ + .flags = SPI_NOR_NO_FR | SPI_NOR_XSR_RDY, + +/** + * struct spi_nor_manufacturer - SPI NOR manufacturer object + * @name: manufacturer name + * @parts: array of parts supported by this manufacturer + * @nparts: number of entries in the parts array + * @fixups: hooks called at various points in time during spi_nor_scan() + */ +struct spi_nor_manufacturer { + const char *name; + const struct flash_info *parts; + unsigned int nparts; + const struct spi_nor_fixups *fixups; +}; + +/* Manufacturer drivers. */ +extern const struct spi_nor_manufacturer spi_nor_atmel; +extern const struct spi_nor_manufacturer spi_nor_catalyst; +extern const struct spi_nor_manufacturer spi_nor_eon; +extern const struct spi_nor_manufacturer spi_nor_esmt; +extern const struct spi_nor_manufacturer spi_nor_everspin; +extern const struct spi_nor_manufacturer spi_nor_fujitsu; +extern const struct spi_nor_manufacturer spi_nor_gigadevice; +extern const struct spi_nor_manufacturer spi_nor_intel; +extern const struct spi_nor_manufacturer spi_nor_issi; +extern const struct spi_nor_manufacturer spi_nor_macronix; +extern const struct spi_nor_manufacturer spi_nor_micron; +extern const struct spi_nor_manufacturer spi_nor_st; +extern const struct spi_nor_manufacturer spi_nor_spansion; +extern const struct spi_nor_manufacturer spi_nor_sst; +extern const struct spi_nor_manufacturer spi_nor_winbond; +extern const struct spi_nor_manufacturer spi_nor_xilinx; +extern const struct spi_nor_manufacturer spi_nor_xmc; + +int spi_nor_write_enable(struct spi_nor *nor); +int spi_nor_write_disable(struct spi_nor *nor); +int spi_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable); +int spi_nor_write_ear(struct spi_nor *nor, u8 ear); +int spi_nor_wait_till_ready(struct spi_nor *nor); +int spi_nor_lock_and_prep(struct spi_nor *nor); +void spi_nor_unlock_and_unprep(struct spi_nor *nor); +int spi_nor_sr1_bit6_quad_enable(struct spi_nor *nor); +int spi_nor_sr2_bit1_quad_enable(struct spi_nor *nor); +int spi_nor_sr2_bit7_quad_enable(struct spi_nor *nor); + +int spi_nor_xread_sr(struct spi_nor *nor, u8 *sr); +ssize_t spi_nor_read_data(struct spi_nor *nor, loff_t from, size_t len, + u8 *buf); +ssize_t spi_nor_write_data(struct spi_nor *nor, loff_t to, size_t len, + const u8 *buf); + +int spi_nor_hwcaps_read2cmd(u32 hwcaps); +u8 spi_nor_convert_3to4_read(u8 opcode); +void spi_nor_set_pp_settings(struct spi_nor_pp_command *pp, u8 opcode, + enum spi_nor_protocol proto); + +void spi_nor_set_erase_type(struct spi_nor_erase_type *erase, u32 size, + u8 opcode); +struct spi_nor_erase_region * +spi_nor_region_next(struct spi_nor_erase_region *region); +void spi_nor_init_uniform_erase_map(struct spi_nor_erase_map *map, + u8 erase_mask, u64 flash_size); + +int spi_nor_post_bfpt_fixups(struct spi_nor *nor, + const struct sfdp_parameter_header *bfpt_header, + const struct sfdp_bfpt *bfpt, + struct spi_nor_flash_parameter *params); + +static struct spi_nor __maybe_unused *mtd_to_spi_nor(struct mtd_info *mtd) +{ + return mtd->priv; +} + +#endif /* __LINUX_MTD_SPI_NOR_INTERNAL_H */ diff --git a/drivers/mtd/spi-nor/eon.c b/drivers/mtd/spi-nor/eon.c new file mode 100644 index 000000000000..ddb8e3650835 --- /dev/null +++ b/drivers/mtd/spi-nor/eon.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info eon_parts[] = { + /* EON -- en25xxx */ + { "en25f32", INFO(0x1c3116, 0, 64 * 1024, 64, SECT_4K) }, + { "en25p32", INFO(0x1c2016, 0, 64 * 1024, 64, 0) }, + { "en25q32b", INFO(0x1c3016, 0, 64 * 1024, 64, 0) }, + { "en25p64", INFO(0x1c2017, 0, 64 * 1024, 128, 0) }, + { "en25q64", INFO(0x1c3017, 0, 64 * 1024, 128, SECT_4K) }, + { "en25q80a", INFO(0x1c3014, 0, 64 * 1024, 16, + SECT_4K | SPI_NOR_DUAL_READ) }, + { "en25qh16", INFO(0x1c7015, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_DUAL_READ) }, + { "en25qh32", INFO(0x1c7016, 0, 64 * 1024, 64, 0) }, + { "en25qh64", INFO(0x1c7017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ) }, + { "en25qh128", INFO(0x1c7018, 0, 64 * 1024, 256, 0) }, + { "en25qh256", INFO(0x1c7019, 0, 64 * 1024, 512, 0) }, + { "en25s64", INFO(0x1c3817, 0, 64 * 1024, 128, SECT_4K) }, +}; + +const struct spi_nor_manufacturer spi_nor_eon = { + .name = "eon", + .parts = eon_parts, + .nparts = ARRAY_SIZE(eon_parts), +}; diff --git a/drivers/mtd/spi-nor/esmt.c b/drivers/mtd/spi-nor/esmt.c new file mode 100644 index 000000000000..c93170008118 --- /dev/null +++ b/drivers/mtd/spi-nor/esmt.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info esmt_parts[] = { + /* ESMT */ + { "f25l32pa", INFO(0x8c2016, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_HAS_LOCK) }, + { "f25l32qa", INFO(0x8c4116, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_HAS_LOCK) }, + { "f25l64qa", INFO(0x8c4117, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_HAS_LOCK) }, +}; + +const struct spi_nor_manufacturer spi_nor_esmt = { + .name = "esmt", + .parts = esmt_parts, + .nparts = ARRAY_SIZE(esmt_parts), +}; diff --git a/drivers/mtd/spi-nor/everspin.c b/drivers/mtd/spi-nor/everspin.c new file mode 100644 index 000000000000..04a177a32283 --- /dev/null +++ b/drivers/mtd/spi-nor/everspin.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info everspin_parts[] = { + /* Everspin */ + { "mr25h128", CAT25_INFO(16 * 1024, 1, 256, 2, + SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, + { "mr25h256", CAT25_INFO(32 * 1024, 1, 256, 2, + SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, + { "mr25h10", CAT25_INFO(128 * 1024, 1, 256, 3, + SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, + { "mr25h40", CAT25_INFO(512 * 1024, 1, 256, 3, + SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, +}; + +const struct spi_nor_manufacturer spi_nor_everspin = { + .name = "everspin", + .parts = everspin_parts, + .nparts = ARRAY_SIZE(everspin_parts), +}; diff --git a/drivers/mtd/spi-nor/fujitsu.c b/drivers/mtd/spi-nor/fujitsu.c new file mode 100644 index 000000000000..e385d93e756c --- /dev/null +++ b/drivers/mtd/spi-nor/fujitsu.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info fujitsu_parts[] = { + /* Fujitsu */ + { "mb85rs1mt", INFO(0x047f27, 0, 128 * 1024, 1, SPI_NOR_NO_ERASE) }, +}; + +const struct spi_nor_manufacturer spi_nor_fujitsu = { + .name = "fujitsu", + .parts = fujitsu_parts, + .nparts = ARRAY_SIZE(fujitsu_parts), +}; diff --git a/drivers/mtd/spi-nor/gigadevice.c b/drivers/mtd/spi-nor/gigadevice.c new file mode 100644 index 000000000000..447d84bb2128 --- /dev/null +++ b/drivers/mtd/spi-nor/gigadevice.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static void gd25q256_default_init(struct spi_nor *nor) +{ + /* + * Some manufacturer like GigaDevice may use different + * bit to set QE on different memories, so the MFR can't + * indicate the quad_enable method for this case, we need + * to set it in the default_init fixup hook. + */ + nor->params->quad_enable = spi_nor_sr1_bit6_quad_enable; +} + +static struct spi_nor_fixups gd25q256_fixups = { + .default_init = gd25q256_default_init, +}; + +static const struct flash_info gigadevice_parts[] = { + { "gd25q16", INFO(0xc84015, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "gd25q32", INFO(0xc84016, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "gd25lq32", INFO(0xc86016, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "gd25q64", INFO(0xc84017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "gd25lq64c", INFO(0xc86017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "gd25lq128d", INFO(0xc86018, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "gd25q128", INFO(0xc84018, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "gd25q256", INFO(0xc84019, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_4B_OPCODES | SPI_NOR_HAS_LOCK | + SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6) + .fixups = &gd25q256_fixups }, +}; + +const struct spi_nor_manufacturer spi_nor_gigadevice = { + .name = "gigadevice", + .parts = gigadevice_parts, + .nparts = ARRAY_SIZE(gigadevice_parts), +}; diff --git a/drivers/mtd/spi-nor/intel.c b/drivers/mtd/spi-nor/intel.c new file mode 100644 index 000000000000..d8196f101368 --- /dev/null +++ b/drivers/mtd/spi-nor/intel.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info intel_parts[] = { + /* Intel/Numonyx -- xxxs33b */ + { "160s33b", INFO(0x898911, 0, 64 * 1024, 32, 0) }, + { "320s33b", INFO(0x898912, 0, 64 * 1024, 64, 0) }, + { "640s33b", INFO(0x898913, 0, 64 * 1024, 128, 0) }, +}; + +static void intel_default_init(struct spi_nor *nor) +{ + nor->flags |= SNOR_F_HAS_LOCK; +} + +static const struct spi_nor_fixups intel_fixups = { + .default_init = intel_default_init, +}; + +const struct spi_nor_manufacturer spi_nor_intel = { + .name = "intel", + .parts = intel_parts, + .nparts = ARRAY_SIZE(intel_parts), + .fixups = &intel_fixups, +}; diff --git a/drivers/mtd/spi-nor/issi.c b/drivers/mtd/spi-nor/issi.c new file mode 100644 index 000000000000..ffcb60e54a80 --- /dev/null +++ b/drivers/mtd/spi-nor/issi.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static int +is25lp256_post_bfpt_fixups(struct spi_nor *nor, + const struct sfdp_parameter_header *bfpt_header, + const struct sfdp_bfpt *bfpt, + struct spi_nor_flash_parameter *params) +{ + /* + * IS25LP256 supports 4B opcodes, but the BFPT advertises a + * BFPT_DWORD1_ADDRESS_BYTES_3_ONLY address width. + * Overwrite the address width advertised by the BFPT. + */ + if ((bfpt->dwords[BFPT_DWORD(1)] & BFPT_DWORD1_ADDRESS_BYTES_MASK) == + BFPT_DWORD1_ADDRESS_BYTES_3_ONLY) + nor->addr_width = 4; + + return 0; +} + +static struct spi_nor_fixups is25lp256_fixups = { + .post_bfpt = is25lp256_post_bfpt_fixups, +}; + +static const struct flash_info issi_parts[] = { + /* ISSI */ + { "is25cd512", INFO(0x7f9d20, 0, 32 * 1024, 2, SECT_4K) }, + { "is25lq040b", INFO(0x9d4013, 0, 64 * 1024, 8, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "is25lp016d", INFO(0x9d6015, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "is25lp080d", INFO(0x9d6014, 0, 64 * 1024, 16, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "is25lp032", INFO(0x9d6016, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ) }, + { "is25lp064", INFO(0x9d6017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ) }, + { "is25lp128", INFO(0x9d6018, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ) }, + { "is25lp256", INFO(0x9d6019, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_4B_OPCODES) + .fixups = &is25lp256_fixups }, + { "is25wp032", INFO(0x9d7016, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "is25wp064", INFO(0x9d7017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "is25wp128", INFO(0x9d7018, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "is25wp256", INFO(0x9d7019, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_4B_OPCODES) + .fixups = &is25lp256_fixups }, + + /* PMC */ + { "pm25lv512", INFO(0, 0, 32 * 1024, 2, SECT_4K_PMC) }, + { "pm25lv010", INFO(0, 0, 32 * 1024, 4, SECT_4K_PMC) }, + { "pm25lq032", INFO(0x7f9d46, 0, 64 * 1024, 64, SECT_4K) }, +}; + +static void issi_default_init(struct spi_nor *nor) +{ + nor->params->quad_enable = spi_nor_sr1_bit6_quad_enable; +} + +static const struct spi_nor_fixups issi_fixups = { + .default_init = issi_default_init, +}; + +const struct spi_nor_manufacturer spi_nor_issi = { + .name = "issi", + .parts = issi_parts, + .nparts = ARRAY_SIZE(issi_parts), + .fixups = &issi_fixups, +}; diff --git a/drivers/mtd/spi-nor/macronix.c b/drivers/mtd/spi-nor/macronix.c new file mode 100644 index 000000000000..ab0f963d630c --- /dev/null +++ b/drivers/mtd/spi-nor/macronix.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static int +mx25l25635_post_bfpt_fixups(struct spi_nor *nor, + const struct sfdp_parameter_header *bfpt_header, + const struct sfdp_bfpt *bfpt, + struct spi_nor_flash_parameter *params) +{ + /* + * MX25L25635F supports 4B opcodes but MX25L25635E does not. + * Unfortunately, Macronix has re-used the same JEDEC ID for both + * variants which prevents us from defining a new entry in the parts + * table. + * We need a way to differentiate MX25L25635E and MX25L25635F, and it + * seems that the F version advertises support for Fast Read 4-4-4 in + * its BFPT table. + */ + if (bfpt->dwords[BFPT_DWORD(5)] & BFPT_DWORD5_FAST_READ_4_4_4) + nor->flags |= SNOR_F_4B_OPCODES; + + return 0; +} + +static struct spi_nor_fixups mx25l25635_fixups = { + .post_bfpt = mx25l25635_post_bfpt_fixups, +}; + +static const struct flash_info macronix_parts[] = { + /* Macronix */ + { "mx25l512e", INFO(0xc22010, 0, 64 * 1024, 1, SECT_4K) }, + { "mx25l2005a", INFO(0xc22012, 0, 64 * 1024, 4, SECT_4K) }, + { "mx25l4005a", INFO(0xc22013, 0, 64 * 1024, 8, SECT_4K) }, + { "mx25l8005", INFO(0xc22014, 0, 64 * 1024, 16, 0) }, + { "mx25l1606e", INFO(0xc22015, 0, 64 * 1024, 32, SECT_4K) }, + { "mx25l3205d", INFO(0xc22016, 0, 64 * 1024, 64, SECT_4K) }, + { "mx25l3255e", INFO(0xc29e16, 0, 64 * 1024, 64, SECT_4K) }, + { "mx25l6405d", INFO(0xc22017, 0, 64 * 1024, 128, SECT_4K) }, + { "mx25u2033e", INFO(0xc22532, 0, 64 * 1024, 4, SECT_4K) }, + { "mx25u3235f", INFO(0xc22536, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ) }, + { "mx25u4035", INFO(0xc22533, 0, 64 * 1024, 8, SECT_4K) }, + { "mx25u8035", INFO(0xc22534, 0, 64 * 1024, 16, SECT_4K) }, + { "mx25u6435f", INFO(0xc22537, 0, 64 * 1024, 128, SECT_4K) }, + { "mx25l12805d", INFO(0xc22018, 0, 64 * 1024, 256, 0) }, + { "mx25l12855e", INFO(0xc22618, 0, 64 * 1024, 256, 0) }, + { "mx25r3235f", INFO(0xc22816, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ) }, + { "mx25u12835f", INFO(0xc22538, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ) }, + { "mx25l25635e", INFO(0xc22019, 0, 64 * 1024, 512, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) + .fixups = &mx25l25635_fixups }, + { "mx25u25635f", INFO(0xc22539, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_4B_OPCODES) }, + { "mx25v8035f", INFO(0xc22314, 0, 64 * 1024, 16, + SECT_4K | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ) }, + { "mx25l25655e", INFO(0xc22619, 0, 64 * 1024, 512, 0) }, + { "mx66l51235l", INFO(0xc2201a, 0, 64 * 1024, 1024, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_4B_OPCODES) }, + { "mx66u51235f", INFO(0xc2253a, 0, 64 * 1024, 1024, + SECT_4K | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, + { "mx66l1g45g", INFO(0xc2201b, 0, 64 * 1024, 2048, + SECT_4K | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ) }, + { "mx66l1g55g", INFO(0xc2261b, 0, 64 * 1024, 2048, + SPI_NOR_QUAD_READ) }, +}; + +static void macronix_default_init(struct spi_nor *nor) +{ + nor->params->quad_enable = spi_nor_sr1_bit6_quad_enable; + nor->params->set_4byte_addr_mode = spi_nor_set_4byte_addr_mode; +} + +static const struct spi_nor_fixups macronix_fixups = { + .default_init = macronix_default_init, +}; + +const struct spi_nor_manufacturer spi_nor_macronix = { + .name = "macronix", + .parts = macronix_parts, + .nparts = ARRAY_SIZE(macronix_parts), + .fixups = ¯onix_fixups, +}; diff --git a/drivers/mtd/spi-nor/micron-st.c b/drivers/mtd/spi-nor/micron-st.c new file mode 100644 index 000000000000..6c034b9718e2 --- /dev/null +++ b/drivers/mtd/spi-nor/micron-st.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info micron_parts[] = { + { "mt35xu512aba", INFO(0x2c5b1a, 0, 128 * 1024, 512, + SECT_4K | USE_FSR | SPI_NOR_OCTAL_READ | + SPI_NOR_4B_OPCODES) }, + { "mt35xu02g", INFO(0x2c5b1c, 0, 128 * 1024, 2048, + SECT_4K | USE_FSR | SPI_NOR_OCTAL_READ | + SPI_NOR_4B_OPCODES) }, +}; + +static const struct flash_info st_parts[] = { + { "n25q016a", INFO(0x20bb15, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_QUAD_READ) }, + { "n25q032", INFO(0x20ba16, 0, 64 * 1024, 64, + SPI_NOR_QUAD_READ) }, + { "n25q032a", INFO(0x20bb16, 0, 64 * 1024, 64, + SPI_NOR_QUAD_READ) }, + { "n25q064", INFO(0x20ba17, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_QUAD_READ) }, + { "n25q064a", INFO(0x20bb17, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_QUAD_READ) }, + { "n25q128a11", INFO(0x20bb18, 0, 64 * 1024, 256, + SECT_4K | USE_FSR | SPI_NOR_QUAD_READ) }, + { "n25q128a13", INFO(0x20ba18, 0, 64 * 1024, 256, + SECT_4K | USE_FSR | SPI_NOR_QUAD_READ) }, + { "mt25ql256a", INFO6(0x20ba19, 0x104400, 64 * 1024, 512, + SECT_4K | USE_FSR | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, + { "n25q256a", INFO(0x20ba19, 0, 64 * 1024, 512, SECT_4K | + USE_FSR | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ) }, + { "mt25qu256a", INFO6(0x20bb19, 0x104400, 64 * 1024, 512, + SECT_4K | USE_FSR | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, + { "n25q256ax1", INFO(0x20bb19, 0, 64 * 1024, 512, + SECT_4K | USE_FSR | SPI_NOR_QUAD_READ) }, + { "mt25ql512a", INFO6(0x20ba20, 0x104400, 64 * 1024, 1024, + SECT_4K | USE_FSR | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, + { "n25q512ax3", INFO(0x20ba20, 0, 64 * 1024, 1024, + SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | + SPI_NOR_4BIT_BP | SPI_NOR_BP3_SR_BIT6) }, + { "mt25qu512a", INFO6(0x20bb20, 0x104400, 64 * 1024, 1024, + SECT_4K | USE_FSR | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, + { "n25q512a", INFO(0x20bb20, 0, 64 * 1024, 1024, + SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | + SPI_NOR_4BIT_BP | SPI_NOR_BP3_SR_BIT6) }, + { "n25q00", INFO(0x20ba21, 0, 64 * 1024, 2048, + SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | + NO_CHIP_ERASE) }, + { "n25q00a", INFO(0x20bb21, 0, 64 * 1024, 2048, + SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | + NO_CHIP_ERASE) }, + { "mt25ql02g", INFO(0x20ba22, 0, 64 * 1024, 4096, + SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | + NO_CHIP_ERASE) }, + { "mt25qu02g", INFO(0x20bb22, 0, 64 * 1024, 4096, + SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | + NO_CHIP_ERASE) }, + + { "m25p05", INFO(0x202010, 0, 32 * 1024, 2, 0) }, + { "m25p10", INFO(0x202011, 0, 32 * 1024, 4, 0) }, + { "m25p20", INFO(0x202012, 0, 64 * 1024, 4, 0) }, + { "m25p40", INFO(0x202013, 0, 64 * 1024, 8, 0) }, + { "m25p80", INFO(0x202014, 0, 64 * 1024, 16, 0) }, + { "m25p16", INFO(0x202015, 0, 64 * 1024, 32, 0) }, + { "m25p32", INFO(0x202016, 0, 64 * 1024, 64, 0) }, + { "m25p64", INFO(0x202017, 0, 64 * 1024, 128, 0) }, + { "m25p128", INFO(0x202018, 0, 256 * 1024, 64, 0) }, + + { "m25p05-nonjedec", INFO(0, 0, 32 * 1024, 2, 0) }, + { "m25p10-nonjedec", INFO(0, 0, 32 * 1024, 4, 0) }, + { "m25p20-nonjedec", INFO(0, 0, 64 * 1024, 4, 0) }, + { "m25p40-nonjedec", INFO(0, 0, 64 * 1024, 8, 0) }, + { "m25p80-nonjedec", INFO(0, 0, 64 * 1024, 16, 0) }, + { "m25p16-nonjedec", INFO(0, 0, 64 * 1024, 32, 0) }, + { "m25p32-nonjedec", INFO(0, 0, 64 * 1024, 64, 0) }, + { "m25p64-nonjedec", INFO(0, 0, 64 * 1024, 128, 0) }, + { "m25p128-nonjedec", INFO(0, 0, 256 * 1024, 64, 0) }, + + { "m45pe10", INFO(0x204011, 0, 64 * 1024, 2, 0) }, + { "m45pe80", INFO(0x204014, 0, 64 * 1024, 16, 0) }, + { "m45pe16", INFO(0x204015, 0, 64 * 1024, 32, 0) }, + + { "m25pe20", INFO(0x208012, 0, 64 * 1024, 4, 0) }, + { "m25pe80", INFO(0x208014, 0, 64 * 1024, 16, 0) }, + { "m25pe16", INFO(0x208015, 0, 64 * 1024, 32, SECT_4K) }, + + { "m25px16", INFO(0x207115, 0, 64 * 1024, 32, SECT_4K) }, + { "m25px32", INFO(0x207116, 0, 64 * 1024, 64, SECT_4K) }, + { "m25px32-s0", INFO(0x207316, 0, 64 * 1024, 64, SECT_4K) }, + { "m25px32-s1", INFO(0x206316, 0, 64 * 1024, 64, SECT_4K) }, + { "m25px64", INFO(0x207117, 0, 64 * 1024, 128, 0) }, + { "m25px80", INFO(0x207114, 0, 64 * 1024, 16, 0) }, +}; + +/** + * st_micron_set_4byte_addr_mode() - Set 4-byte address mode for ST and Micron + * flashes. + * @nor: pointer to 'struct spi_nor'. + * @enable: true to enter the 4-byte address mode, false to exit the 4-byte + * address mode. + * + * Return: 0 on success, -errno otherwise. + */ +static int st_micron_set_4byte_addr_mode(struct spi_nor *nor, bool enable) +{ + int ret; + + ret = spi_nor_write_enable(nor); + if (ret) + return ret; + + ret = spi_nor_set_4byte_addr_mode(nor, enable); + if (ret) + return ret; + + return spi_nor_write_disable(nor); +} + +static void micron_st_default_init(struct spi_nor *nor) +{ + nor->flags |= SNOR_F_HAS_LOCK; + nor->flags &= ~SNOR_F_HAS_16BIT_SR; + nor->params->quad_enable = NULL; + nor->params->set_4byte_addr_mode = st_micron_set_4byte_addr_mode; +} + +static const struct spi_nor_fixups micron_st_fixups = { + .default_init = micron_st_default_init, +}; + +const struct spi_nor_manufacturer spi_nor_micron = { + .name = "micron", + .parts = micron_parts, + .nparts = ARRAY_SIZE(micron_parts), + .fixups = µn_st_fixups, +}; + +const struct spi_nor_manufacturer spi_nor_st = { + .name = "st", + .parts = st_parts, + .nparts = ARRAY_SIZE(st_parts), + .fixups = µn_st_fixups, +}; diff --git a/drivers/mtd/spi-nor/sfdp.c b/drivers/mtd/spi-nor/sfdp.c new file mode 100644 index 000000000000..f6038d3a3684 --- /dev/null +++ b/drivers/mtd/spi-nor/sfdp.c @@ -0,0 +1,1204 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include +#include +#include + +#include "core.h" + +#define SFDP_PARAM_HEADER_ID(p) (((p)->id_msb << 8) | (p)->id_lsb) +#define SFDP_PARAM_HEADER_PTP(p) \ + (((p)->parameter_table_pointer[2] << 16) | \ + ((p)->parameter_table_pointer[1] << 8) | \ + ((p)->parameter_table_pointer[0] << 0)) + +#define SFDP_BFPT_ID 0xff00 /* Basic Flash Parameter Table */ +#define SFDP_SECTOR_MAP_ID 0xff81 /* Sector Map Table */ +#define SFDP_4BAIT_ID 0xff84 /* 4-byte Address Instruction Table */ + +#define SFDP_SIGNATURE 0x50444653U +#define SFDP_JESD216_MAJOR 1 +#define SFDP_JESD216_MINOR 0 +#define SFDP_JESD216A_MINOR 5 +#define SFDP_JESD216B_MINOR 6 + +struct sfdp_header { + u32 signature; /* Ox50444653U <=> "SFDP" */ + u8 minor; + u8 major; + u8 nph; /* 0-base number of parameter headers */ + u8 unused; + + /* Basic Flash Parameter Table. */ + struct sfdp_parameter_header bfpt_header; +}; + +/* Fast Read settings. */ +struct sfdp_bfpt_read { + /* The Fast Read x-y-z hardware capability in params->hwcaps.mask. */ + u32 hwcaps; + + /* + * The bit in BFPT DWORD tells us + * whether the Fast Read x-y-z command is supported. + */ + u32 supported_dword; + u32 supported_bit; + + /* + * The half-word at offset in BFPT DWORD + * encodes the op code, the number of mode clocks and the number of wait + * states to be used by Fast Read x-y-z command. + */ + u32 settings_dword; + u32 settings_shift; + + /* The SPI protocol for this Fast Read x-y-z command. */ + enum spi_nor_protocol proto; +}; + +struct sfdp_bfpt_erase { + /* + * The half-word at offset in DWORD encodes the + * op code and erase sector size to be used by Sector Erase commands. + */ + u32 dword; + u32 shift; +}; + +#define SMPT_CMD_ADDRESS_LEN_MASK GENMASK(23, 22) +#define SMPT_CMD_ADDRESS_LEN_0 (0x0UL << 22) +#define SMPT_CMD_ADDRESS_LEN_3 (0x1UL << 22) +#define SMPT_CMD_ADDRESS_LEN_4 (0x2UL << 22) +#define SMPT_CMD_ADDRESS_LEN_USE_CURRENT (0x3UL << 22) + +#define SMPT_CMD_READ_DUMMY_MASK GENMASK(19, 16) +#define SMPT_CMD_READ_DUMMY_SHIFT 16 +#define SMPT_CMD_READ_DUMMY(_cmd) \ + (((_cmd) & SMPT_CMD_READ_DUMMY_MASK) >> SMPT_CMD_READ_DUMMY_SHIFT) +#define SMPT_CMD_READ_DUMMY_IS_VARIABLE 0xfUL + +#define SMPT_CMD_READ_DATA_MASK GENMASK(31, 24) +#define SMPT_CMD_READ_DATA_SHIFT 24 +#define SMPT_CMD_READ_DATA(_cmd) \ + (((_cmd) & SMPT_CMD_READ_DATA_MASK) >> SMPT_CMD_READ_DATA_SHIFT) + +#define SMPT_CMD_OPCODE_MASK GENMASK(15, 8) +#define SMPT_CMD_OPCODE_SHIFT 8 +#define SMPT_CMD_OPCODE(_cmd) \ + (((_cmd) & SMPT_CMD_OPCODE_MASK) >> SMPT_CMD_OPCODE_SHIFT) + +#define SMPT_MAP_REGION_COUNT_MASK GENMASK(23, 16) +#define SMPT_MAP_REGION_COUNT_SHIFT 16 +#define SMPT_MAP_REGION_COUNT(_header) \ + ((((_header) & SMPT_MAP_REGION_COUNT_MASK) >> \ + SMPT_MAP_REGION_COUNT_SHIFT) + 1) + +#define SMPT_MAP_ID_MASK GENMASK(15, 8) +#define SMPT_MAP_ID_SHIFT 8 +#define SMPT_MAP_ID(_header) \ + (((_header) & SMPT_MAP_ID_MASK) >> SMPT_MAP_ID_SHIFT) + +#define SMPT_MAP_REGION_SIZE_MASK GENMASK(31, 8) +#define SMPT_MAP_REGION_SIZE_SHIFT 8 +#define SMPT_MAP_REGION_SIZE(_region) \ + (((((_region) & SMPT_MAP_REGION_SIZE_MASK) >> \ + SMPT_MAP_REGION_SIZE_SHIFT) + 1) * 256) + +#define SMPT_MAP_REGION_ERASE_TYPE_MASK GENMASK(3, 0) +#define SMPT_MAP_REGION_ERASE_TYPE(_region) \ + ((_region) & SMPT_MAP_REGION_ERASE_TYPE_MASK) + +#define SMPT_DESC_TYPE_MAP BIT(1) +#define SMPT_DESC_END BIT(0) + +#define SFDP_4BAIT_DWORD_MAX 2 + +struct sfdp_4bait { + /* The hardware capability. */ + u32 hwcaps; + + /* + * The bit in DWORD1 of the 4BAIT tells us whether + * the associated 4-byte address op code is supported. + */ + u32 supported_bit; +}; + +/** + * spi_nor_read_raw() - raw read of serial flash memory. read_opcode, + * addr_width and read_dummy members of the struct spi_nor + * should be previously + * set. + * @nor: pointer to a 'struct spi_nor' + * @addr: offset in the serial flash memory + * @len: number of bytes to read + * @buf: buffer where the data is copied into (dma-safe memory) + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_read_raw(struct spi_nor *nor, u32 addr, size_t len, u8 *buf) +{ + ssize_t ret; + + while (len) { + ret = spi_nor_read_data(nor, addr, len, buf); + if (ret < 0) + return ret; + if (!ret || ret > len) + return -EIO; + + buf += ret; + addr += ret; + len -= ret; + } + return 0; +} + +/** + * spi_nor_read_sfdp() - read Serial Flash Discoverable Parameters. + * @nor: pointer to a 'struct spi_nor' + * @addr: offset in the SFDP area to start reading data from + * @len: number of bytes to read + * @buf: buffer where the SFDP data are copied into (dma-safe memory) + * + * Whatever the actual numbers of bytes for address and dummy cycles are + * for (Fast) Read commands, the Read SFDP (5Ah) instruction is always + * followed by a 3-byte address and 8 dummy clock cycles. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_read_sfdp(struct spi_nor *nor, u32 addr, + size_t len, void *buf) +{ + u8 addr_width, read_opcode, read_dummy; + int ret; + + read_opcode = nor->read_opcode; + addr_width = nor->addr_width; + read_dummy = nor->read_dummy; + + nor->read_opcode = SPINOR_OP_RDSFDP; + nor->addr_width = 3; + nor->read_dummy = 8; + + ret = spi_nor_read_raw(nor, addr, len, buf); + + nor->read_opcode = read_opcode; + nor->addr_width = addr_width; + nor->read_dummy = read_dummy; + + return ret; +} + +/** + * spi_nor_read_sfdp_dma_unsafe() - read Serial Flash Discoverable Parameters. + * @nor: pointer to a 'struct spi_nor' + * @addr: offset in the SFDP area to start reading data from + * @len: number of bytes to read + * @buf: buffer where the SFDP data are copied into + * + * Wrap spi_nor_read_sfdp() using a kmalloc'ed bounce buffer as @buf is now not + * guaranteed to be dma-safe. + * + * Return: -ENOMEM if kmalloc() fails, the return code of spi_nor_read_sfdp() + * otherwise. + */ +static int spi_nor_read_sfdp_dma_unsafe(struct spi_nor *nor, u32 addr, + size_t len, void *buf) +{ + void *dma_safe_buf; + int ret; + + dma_safe_buf = kmalloc(len, GFP_KERNEL); + if (!dma_safe_buf) + return -ENOMEM; + + ret = spi_nor_read_sfdp(nor, addr, len, dma_safe_buf); + memcpy(buf, dma_safe_buf, len); + kfree(dma_safe_buf); + + return ret; +} + +static void +spi_nor_set_read_settings_from_bfpt(struct spi_nor_read_command *read, + u16 half, + enum spi_nor_protocol proto) +{ + read->num_mode_clocks = (half >> 5) & 0x07; + read->num_wait_states = (half >> 0) & 0x1f; + read->opcode = (half >> 8) & 0xff; + read->proto = proto; +} + +static const struct sfdp_bfpt_read sfdp_bfpt_reads[] = { + /* Fast Read 1-1-2 */ + { + SNOR_HWCAPS_READ_1_1_2, + BFPT_DWORD(1), BIT(16), /* Supported bit */ + BFPT_DWORD(4), 0, /* Settings */ + SNOR_PROTO_1_1_2, + }, + + /* Fast Read 1-2-2 */ + { + SNOR_HWCAPS_READ_1_2_2, + BFPT_DWORD(1), BIT(20), /* Supported bit */ + BFPT_DWORD(4), 16, /* Settings */ + SNOR_PROTO_1_2_2, + }, + + /* Fast Read 2-2-2 */ + { + SNOR_HWCAPS_READ_2_2_2, + BFPT_DWORD(5), BIT(0), /* Supported bit */ + BFPT_DWORD(6), 16, /* Settings */ + SNOR_PROTO_2_2_2, + }, + + /* Fast Read 1-1-4 */ + { + SNOR_HWCAPS_READ_1_1_4, + BFPT_DWORD(1), BIT(22), /* Supported bit */ + BFPT_DWORD(3), 16, /* Settings */ + SNOR_PROTO_1_1_4, + }, + + /* Fast Read 1-4-4 */ + { + SNOR_HWCAPS_READ_1_4_4, + BFPT_DWORD(1), BIT(21), /* Supported bit */ + BFPT_DWORD(3), 0, /* Settings */ + SNOR_PROTO_1_4_4, + }, + + /* Fast Read 4-4-4 */ + { + SNOR_HWCAPS_READ_4_4_4, + BFPT_DWORD(5), BIT(4), /* Supported bit */ + BFPT_DWORD(7), 16, /* Settings */ + SNOR_PROTO_4_4_4, + }, +}; + +static const struct sfdp_bfpt_erase sfdp_bfpt_erases[] = { + /* Erase Type 1 in DWORD8 bits[15:0] */ + {BFPT_DWORD(8), 0}, + + /* Erase Type 2 in DWORD8 bits[31:16] */ + {BFPT_DWORD(8), 16}, + + /* Erase Type 3 in DWORD9 bits[15:0] */ + {BFPT_DWORD(9), 0}, + + /* Erase Type 4 in DWORD9 bits[31:16] */ + {BFPT_DWORD(9), 16}, +}; + +/** + * spi_nor_set_erase_settings_from_bfpt() - set erase type settings from BFPT + * @erase: pointer to a structure that describes a SPI NOR erase type + * @size: the size of the sector/block erased by the erase type + * @opcode: the SPI command op code to erase the sector/block + * @i: erase type index as sorted in the Basic Flash Parameter Table + * + * The supported Erase Types will be sorted at init in ascending order, with + * the smallest Erase Type size being the first member in the erase_type array + * of the spi_nor_erase_map structure. Save the Erase Type index as sorted in + * the Basic Flash Parameter Table since it will be used later on to + * synchronize with the supported Erase Types defined in SFDP optional tables. + */ +static void +spi_nor_set_erase_settings_from_bfpt(struct spi_nor_erase_type *erase, + u32 size, u8 opcode, u8 i) +{ + erase->idx = i; + spi_nor_set_erase_type(erase, size, opcode); +} + +/** + * spi_nor_map_cmp_erase_type() - compare the map's erase types by size + * @l: member in the left half of the map's erase_type array + * @r: member in the right half of the map's erase_type array + * + * Comparison function used in the sort() call to sort in ascending order the + * map's erase types, the smallest erase type size being the first member in the + * sorted erase_type array. + * + * Return: the result of @l->size - @r->size + */ +static int spi_nor_map_cmp_erase_type(const void *l, const void *r) +{ + const struct spi_nor_erase_type *left = l, *right = r; + + return left->size - right->size; +} + +/** + * spi_nor_sort_erase_mask() - sort erase mask + * @map: the erase map of the SPI NOR + * @erase_mask: the erase type mask to be sorted + * + * Replicate the sort done for the map's erase types in BFPT: sort the erase + * mask in ascending order with the smallest erase type size starting from + * BIT(0) in the sorted erase mask. + * + * Return: sorted erase mask. + */ +static u8 spi_nor_sort_erase_mask(struct spi_nor_erase_map *map, u8 erase_mask) +{ + struct spi_nor_erase_type *erase_type = map->erase_type; + int i; + u8 sorted_erase_mask = 0; + + if (!erase_mask) + return 0; + + /* Replicate the sort done for the map's erase types. */ + for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) + if (erase_type[i].size && erase_mask & BIT(erase_type[i].idx)) + sorted_erase_mask |= BIT(i); + + return sorted_erase_mask; +} + +/** + * spi_nor_regions_sort_erase_types() - sort erase types in each region + * @map: the erase map of the SPI NOR + * + * Function assumes that the erase types defined in the erase map are already + * sorted in ascending order, with the smallest erase type size being the first + * member in the erase_type array. It replicates the sort done for the map's + * erase types. Each region's erase bitmask will indicate which erase types are + * supported from the sorted erase types defined in the erase map. + * Sort the all region's erase type at init in order to speed up the process of + * finding the best erase command at runtime. + */ +static void spi_nor_regions_sort_erase_types(struct spi_nor_erase_map *map) +{ + struct spi_nor_erase_region *region = map->regions; + u8 region_erase_mask, sorted_erase_mask; + + while (region) { + region_erase_mask = region->offset & SNOR_ERASE_TYPE_MASK; + + sorted_erase_mask = spi_nor_sort_erase_mask(map, + region_erase_mask); + + /* Overwrite erase mask. */ + region->offset = (region->offset & ~SNOR_ERASE_TYPE_MASK) | + sorted_erase_mask; + + region = spi_nor_region_next(region); + } +} + +/** + * spi_nor_parse_bfpt() - read and parse the Basic Flash Parameter Table. + * @nor: pointer to a 'struct spi_nor' + * @bfpt_header: pointer to the 'struct sfdp_parameter_header' describing + * the Basic Flash Parameter Table length and version + * @params: pointer to the 'struct spi_nor_flash_parameter' to be + * filled + * + * The Basic Flash Parameter Table is the main and only mandatory table as + * defined by the SFDP (JESD216) specification. + * It provides us with the total size (memory density) of the data array and + * the number of address bytes for Fast Read, Page Program and Sector Erase + * commands. + * For Fast READ commands, it also gives the number of mode clock cycles and + * wait states (regrouped in the number of dummy clock cycles) for each + * supported instruction op code. + * For Page Program, the page size is now available since JESD216 rev A, however + * the supported instruction op codes are still not provided. + * For Sector Erase commands, this table stores the supported instruction op + * codes and the associated sector sizes. + * Finally, the Quad Enable Requirements (QER) are also available since JESD216 + * rev A. The QER bits encode the manufacturer dependent procedure to be + * executed to set the Quad Enable (QE) bit in some internal register of the + * Quad SPI memory. Indeed the QE bit, when it exists, must be set before + * sending any Quad SPI command to the memory. Actually, setting the QE bit + * tells the memory to reassign its WP# and HOLD#/RESET# pins to functions IO2 + * and IO3 hence enabling 4 (Quad) I/O lines. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_parse_bfpt(struct spi_nor *nor, + const struct sfdp_parameter_header *bfpt_header, + struct spi_nor_flash_parameter *params) +{ + struct spi_nor_erase_map *map = ¶ms->erase_map; + struct spi_nor_erase_type *erase_type = map->erase_type; + struct sfdp_bfpt bfpt; + size_t len; + int i, cmd, err; + u32 addr; + u16 half; + u8 erase_mask; + + /* JESD216 Basic Flash Parameter Table length is at least 9 DWORDs. */ + if (bfpt_header->length < BFPT_DWORD_MAX_JESD216) + return -EINVAL; + + /* Read the Basic Flash Parameter Table. */ + len = min_t(size_t, sizeof(bfpt), + bfpt_header->length * sizeof(u32)); + addr = SFDP_PARAM_HEADER_PTP(bfpt_header); + memset(&bfpt, 0, sizeof(bfpt)); + err = spi_nor_read_sfdp_dma_unsafe(nor, addr, len, &bfpt); + if (err < 0) + return err; + + /* Fix endianness of the BFPT DWORDs. */ + le32_to_cpu_array(bfpt.dwords, BFPT_DWORD_MAX); + + /* Number of address bytes. */ + switch (bfpt.dwords[BFPT_DWORD(1)] & BFPT_DWORD1_ADDRESS_BYTES_MASK) { + case BFPT_DWORD1_ADDRESS_BYTES_3_ONLY: + nor->addr_width = 3; + break; + + case BFPT_DWORD1_ADDRESS_BYTES_4_ONLY: + nor->addr_width = 4; + break; + + default: + break; + } + + /* Flash Memory Density (in bits). */ + params->size = bfpt.dwords[BFPT_DWORD(2)]; + if (params->size & BIT(31)) { + params->size &= ~BIT(31); + + /* + * Prevent overflows on params->size. Anyway, a NOR of 2^64 + * bits is unlikely to exist so this error probably means + * the BFPT we are reading is corrupted/wrong. + */ + if (params->size > 63) + return -EINVAL; + + params->size = 1ULL << params->size; + } else { + params->size++; + } + params->size >>= 3; /* Convert to bytes. */ + + /* Fast Read settings. */ + for (i = 0; i < ARRAY_SIZE(sfdp_bfpt_reads); i++) { + const struct sfdp_bfpt_read *rd = &sfdp_bfpt_reads[i]; + struct spi_nor_read_command *read; + + if (!(bfpt.dwords[rd->supported_dword] & rd->supported_bit)) { + params->hwcaps.mask &= ~rd->hwcaps; + continue; + } + + params->hwcaps.mask |= rd->hwcaps; + cmd = spi_nor_hwcaps_read2cmd(rd->hwcaps); + read = ¶ms->reads[cmd]; + half = bfpt.dwords[rd->settings_dword] >> rd->settings_shift; + spi_nor_set_read_settings_from_bfpt(read, half, rd->proto); + } + + /* + * Sector Erase settings. Reinitialize the uniform erase map using the + * Erase Types defined in the bfpt table. + */ + erase_mask = 0; + memset(¶ms->erase_map, 0, sizeof(params->erase_map)); + for (i = 0; i < ARRAY_SIZE(sfdp_bfpt_erases); i++) { + const struct sfdp_bfpt_erase *er = &sfdp_bfpt_erases[i]; + u32 erasesize; + u8 opcode; + + half = bfpt.dwords[er->dword] >> er->shift; + erasesize = half & 0xff; + + /* erasesize == 0 means this Erase Type is not supported. */ + if (!erasesize) + continue; + + erasesize = 1U << erasesize; + opcode = (half >> 8) & 0xff; + erase_mask |= BIT(i); + spi_nor_set_erase_settings_from_bfpt(&erase_type[i], erasesize, + opcode, i); + } + spi_nor_init_uniform_erase_map(map, erase_mask, params->size); + /* + * Sort all the map's Erase Types in ascending order with the smallest + * erase size being the first member in the erase_type array. + */ + sort(erase_type, SNOR_ERASE_TYPE_MAX, sizeof(erase_type[0]), + spi_nor_map_cmp_erase_type, NULL); + /* + * Sort the erase types in the uniform region in order to update the + * uniform_erase_type bitmask. The bitmask will be used later on when + * selecting the uniform erase. + */ + spi_nor_regions_sort_erase_types(map); + map->uniform_erase_type = map->uniform_region.offset & + SNOR_ERASE_TYPE_MASK; + + /* Stop here if not JESD216 rev A or later. */ + if (bfpt_header->length < BFPT_DWORD_MAX) + return spi_nor_post_bfpt_fixups(nor, bfpt_header, &bfpt, + params); + + /* Page size: this field specifies 'N' so the page size = 2^N bytes. */ + params->page_size = bfpt.dwords[BFPT_DWORD(11)]; + params->page_size &= BFPT_DWORD11_PAGE_SIZE_MASK; + params->page_size >>= BFPT_DWORD11_PAGE_SIZE_SHIFT; + params->page_size = 1U << params->page_size; + + /* Quad Enable Requirements. */ + switch (bfpt.dwords[BFPT_DWORD(15)] & BFPT_DWORD15_QER_MASK) { + case BFPT_DWORD15_QER_NONE: + params->quad_enable = NULL; + break; + + case BFPT_DWORD15_QER_SR2_BIT1_BUGGY: + /* + * Writing only one byte to the Status Register has the + * side-effect of clearing Status Register 2. + */ + case BFPT_DWORD15_QER_SR2_BIT1_NO_RD: + /* + * Read Configuration Register (35h) instruction is not + * supported. + */ + nor->flags |= SNOR_F_HAS_16BIT_SR | SNOR_F_NO_READ_CR; + params->quad_enable = spi_nor_sr2_bit1_quad_enable; + break; + + case BFPT_DWORD15_QER_SR1_BIT6: + nor->flags &= ~SNOR_F_HAS_16BIT_SR; + params->quad_enable = spi_nor_sr1_bit6_quad_enable; + break; + + case BFPT_DWORD15_QER_SR2_BIT7: + nor->flags &= ~SNOR_F_HAS_16BIT_SR; + params->quad_enable = spi_nor_sr2_bit7_quad_enable; + break; + + case BFPT_DWORD15_QER_SR2_BIT1: + /* + * JESD216 rev B or later does not specify if writing only one + * byte to the Status Register clears or not the Status + * Register 2, so let's be cautious and keep the default + * assumption of a 16-bit Write Status (01h) command. + */ + nor->flags |= SNOR_F_HAS_16BIT_SR; + + params->quad_enable = spi_nor_sr2_bit1_quad_enable; + break; + + default: + return -EINVAL; + } + + return spi_nor_post_bfpt_fixups(nor, bfpt_header, &bfpt, params); +} + +/** + * spi_nor_smpt_addr_width() - return the address width used in the + * configuration detection command. + * @nor: pointer to a 'struct spi_nor' + * @settings: configuration detection command descriptor, dword1 + */ +static u8 spi_nor_smpt_addr_width(const struct spi_nor *nor, const u32 settings) +{ + switch (settings & SMPT_CMD_ADDRESS_LEN_MASK) { + case SMPT_CMD_ADDRESS_LEN_0: + return 0; + case SMPT_CMD_ADDRESS_LEN_3: + return 3; + case SMPT_CMD_ADDRESS_LEN_4: + return 4; + case SMPT_CMD_ADDRESS_LEN_USE_CURRENT: + default: + return nor->addr_width; + } +} + +/** + * spi_nor_smpt_read_dummy() - return the configuration detection command read + * latency, in clock cycles. + * @nor: pointer to a 'struct spi_nor' + * @settings: configuration detection command descriptor, dword1 + * + * Return: the number of dummy cycles for an SMPT read + */ +static u8 spi_nor_smpt_read_dummy(const struct spi_nor *nor, const u32 settings) +{ + u8 read_dummy = SMPT_CMD_READ_DUMMY(settings); + + if (read_dummy == SMPT_CMD_READ_DUMMY_IS_VARIABLE) + return nor->read_dummy; + return read_dummy; +} + +/** + * spi_nor_get_map_in_use() - get the configuration map in use + * @nor: pointer to a 'struct spi_nor' + * @smpt: pointer to the sector map parameter table + * @smpt_len: sector map parameter table length + * + * Return: pointer to the map in use, ERR_PTR(-errno) otherwise. + */ +static const u32 *spi_nor_get_map_in_use(struct spi_nor *nor, const u32 *smpt, + u8 smpt_len) +{ + const u32 *ret; + u8 *buf; + u32 addr; + int err; + u8 i; + u8 addr_width, read_opcode, read_dummy; + u8 read_data_mask, map_id; + + /* Use a kmalloc'ed bounce buffer to guarantee it is DMA-able. */ + buf = kmalloc(sizeof(*buf), GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + + addr_width = nor->addr_width; + read_dummy = nor->read_dummy; + read_opcode = nor->read_opcode; + + map_id = 0; + /* Determine if there are any optional Detection Command Descriptors */ + for (i = 0; i < smpt_len; i += 2) { + if (smpt[i] & SMPT_DESC_TYPE_MAP) + break; + + read_data_mask = SMPT_CMD_READ_DATA(smpt[i]); + nor->addr_width = spi_nor_smpt_addr_width(nor, smpt[i]); + nor->read_dummy = spi_nor_smpt_read_dummy(nor, smpt[i]); + nor->read_opcode = SMPT_CMD_OPCODE(smpt[i]); + addr = smpt[i + 1]; + + err = spi_nor_read_raw(nor, addr, 1, buf); + if (err) { + ret = ERR_PTR(err); + goto out; + } + + /* + * Build an index value that is used to select the Sector Map + * Configuration that is currently in use. + */ + map_id = map_id << 1 | !!(*buf & read_data_mask); + } + + /* + * If command descriptors are provided, they always precede map + * descriptors in the table. There is no need to start the iteration + * over smpt array all over again. + * + * Find the matching configuration map. + */ + ret = ERR_PTR(-EINVAL); + while (i < smpt_len) { + if (SMPT_MAP_ID(smpt[i]) == map_id) { + ret = smpt + i; + break; + } + + /* + * If there are no more configuration map descriptors and no + * configuration ID matched the configuration identifier, the + * sector address map is unknown. + */ + if (smpt[i] & SMPT_DESC_END) + break; + + /* increment the table index to the next map */ + i += SMPT_MAP_REGION_COUNT(smpt[i]) + 1; + } + + /* fall through */ +out: + kfree(buf); + nor->addr_width = addr_width; + nor->read_dummy = read_dummy; + nor->read_opcode = read_opcode; + return ret; +} + +static void spi_nor_region_mark_end(struct spi_nor_erase_region *region) +{ + region->offset |= SNOR_LAST_REGION; +} + +static void spi_nor_region_mark_overlay(struct spi_nor_erase_region *region) +{ + region->offset |= SNOR_OVERLAID_REGION; +} + +/** + * spi_nor_region_check_overlay() - set overlay bit when the region is overlaid + * @region: pointer to a structure that describes a SPI NOR erase region + * @erase: pointer to a structure that describes a SPI NOR erase type + * @erase_type: erase type bitmask + */ +static void +spi_nor_region_check_overlay(struct spi_nor_erase_region *region, + const struct spi_nor_erase_type *erase, + const u8 erase_type) +{ + int i; + + for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { + if (!(erase_type & BIT(i))) + continue; + if (region->size & erase[i].size_mask) { + spi_nor_region_mark_overlay(region); + return; + } + } +} + +/** + * spi_nor_init_non_uniform_erase_map() - initialize the non-uniform erase map + * @nor: pointer to a 'struct spi_nor' + * @params: pointer to a duplicate 'struct spi_nor_flash_parameter' that is + * used for storing SFDP parsed data + * @smpt: pointer to the sector map parameter table + * + * Return: 0 on success, -errno otherwise. + */ +static int +spi_nor_init_non_uniform_erase_map(struct spi_nor *nor, + struct spi_nor_flash_parameter *params, + const u32 *smpt) +{ + struct spi_nor_erase_map *map = ¶ms->erase_map; + struct spi_nor_erase_type *erase = map->erase_type; + struct spi_nor_erase_region *region; + u64 offset; + u32 region_count; + int i, j; + u8 uniform_erase_type, save_uniform_erase_type; + u8 erase_type, regions_erase_type; + + region_count = SMPT_MAP_REGION_COUNT(*smpt); + /* + * The regions will be freed when the driver detaches from the + * device. + */ + region = devm_kcalloc(nor->dev, region_count, sizeof(*region), + GFP_KERNEL); + if (!region) + return -ENOMEM; + map->regions = region; + + uniform_erase_type = 0xff; + regions_erase_type = 0; + offset = 0; + /* Populate regions. */ + for (i = 0; i < region_count; i++) { + j = i + 1; /* index for the region dword */ + region[i].size = SMPT_MAP_REGION_SIZE(smpt[j]); + erase_type = SMPT_MAP_REGION_ERASE_TYPE(smpt[j]); + region[i].offset = offset | erase_type; + + spi_nor_region_check_overlay(®ion[i], erase, erase_type); + + /* + * Save the erase types that are supported in all regions and + * can erase the entire flash memory. + */ + uniform_erase_type &= erase_type; + + /* + * regions_erase_type mask will indicate all the erase types + * supported in this configuration map. + */ + regions_erase_type |= erase_type; + + offset = (region[i].offset & ~SNOR_ERASE_FLAGS_MASK) + + region[i].size; + } + + save_uniform_erase_type = map->uniform_erase_type; + map->uniform_erase_type = spi_nor_sort_erase_mask(map, + uniform_erase_type); + + if (!regions_erase_type) { + /* + * Roll back to the previous uniform_erase_type mask, SMPT is + * broken. + */ + map->uniform_erase_type = save_uniform_erase_type; + return -EINVAL; + } + + /* + * BFPT advertises all the erase types supported by all the possible + * map configurations. Mask out the erase types that are not supported + * by the current map configuration. + */ + for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) + if (!(regions_erase_type & BIT(erase[i].idx))) + spi_nor_set_erase_type(&erase[i], 0, 0xFF); + + spi_nor_region_mark_end(®ion[i - 1]); + + return 0; +} + +/** + * spi_nor_parse_smpt() - parse Sector Map Parameter Table + * @nor: pointer to a 'struct spi_nor' + * @smpt_header: sector map parameter table header + * @params: pointer to a duplicate 'struct spi_nor_flash_parameter' + * that is used for storing SFDP parsed data + * + * This table is optional, but when available, we parse it to identify the + * location and size of sectors within the main data array of the flash memory + * device and to identify which Erase Types are supported by each sector. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_parse_smpt(struct spi_nor *nor, + const struct sfdp_parameter_header *smpt_header, + struct spi_nor_flash_parameter *params) +{ + const u32 *sector_map; + u32 *smpt; + size_t len; + u32 addr; + int ret; + + /* Read the Sector Map Parameter Table. */ + len = smpt_header->length * sizeof(*smpt); + smpt = kmalloc(len, GFP_KERNEL); + if (!smpt) + return -ENOMEM; + + addr = SFDP_PARAM_HEADER_PTP(smpt_header); + ret = spi_nor_read_sfdp(nor, addr, len, smpt); + if (ret) + goto out; + + /* Fix endianness of the SMPT DWORDs. */ + le32_to_cpu_array(smpt, smpt_header->length); + + sector_map = spi_nor_get_map_in_use(nor, smpt, smpt_header->length); + if (IS_ERR(sector_map)) { + ret = PTR_ERR(sector_map); + goto out; + } + + ret = spi_nor_init_non_uniform_erase_map(nor, params, sector_map); + if (ret) + goto out; + + spi_nor_regions_sort_erase_types(¶ms->erase_map); + /* fall through */ +out: + kfree(smpt); + return ret; +} + +/** + * spi_nor_parse_4bait() - parse the 4-Byte Address Instruction Table + * @nor: pointer to a 'struct spi_nor'. + * @param_header: pointer to the 'struct sfdp_parameter_header' describing + * the 4-Byte Address Instruction Table length and version. + * @params: pointer to the 'struct spi_nor_flash_parameter' to be. + * + * Return: 0 on success, -errno otherwise. + */ +static int spi_nor_parse_4bait(struct spi_nor *nor, + const struct sfdp_parameter_header *param_header, + struct spi_nor_flash_parameter *params) +{ + static const struct sfdp_4bait reads[] = { + { SNOR_HWCAPS_READ, BIT(0) }, + { SNOR_HWCAPS_READ_FAST, BIT(1) }, + { SNOR_HWCAPS_READ_1_1_2, BIT(2) }, + { SNOR_HWCAPS_READ_1_2_2, BIT(3) }, + { SNOR_HWCAPS_READ_1_1_4, BIT(4) }, + { SNOR_HWCAPS_READ_1_4_4, BIT(5) }, + { SNOR_HWCAPS_READ_1_1_1_DTR, BIT(13) }, + { SNOR_HWCAPS_READ_1_2_2_DTR, BIT(14) }, + { SNOR_HWCAPS_READ_1_4_4_DTR, BIT(15) }, + }; + static const struct sfdp_4bait programs[] = { + { SNOR_HWCAPS_PP, BIT(6) }, + { SNOR_HWCAPS_PP_1_1_4, BIT(7) }, + { SNOR_HWCAPS_PP_1_4_4, BIT(8) }, + }; + static const struct sfdp_4bait erases[SNOR_ERASE_TYPE_MAX] = { + { 0u /* not used */, BIT(9) }, + { 0u /* not used */, BIT(10) }, + { 0u /* not used */, BIT(11) }, + { 0u /* not used */, BIT(12) }, + }; + struct spi_nor_pp_command *params_pp = params->page_programs; + struct spi_nor_erase_map *map = ¶ms->erase_map; + struct spi_nor_erase_type *erase_type = map->erase_type; + u32 *dwords; + size_t len; + u32 addr, discard_hwcaps, read_hwcaps, pp_hwcaps, erase_mask; + int i, ret; + + if (param_header->major != SFDP_JESD216_MAJOR || + param_header->length < SFDP_4BAIT_DWORD_MAX) + return -EINVAL; + + /* Read the 4-byte Address Instruction Table. */ + len = sizeof(*dwords) * SFDP_4BAIT_DWORD_MAX; + + /* Use a kmalloc'ed bounce buffer to guarantee it is DMA-able. */ + dwords = kmalloc(len, GFP_KERNEL); + if (!dwords) + return -ENOMEM; + + addr = SFDP_PARAM_HEADER_PTP(param_header); + ret = spi_nor_read_sfdp(nor, addr, len, dwords); + if (ret) + goto out; + + /* Fix endianness of the 4BAIT DWORDs. */ + le32_to_cpu_array(dwords, SFDP_4BAIT_DWORD_MAX); + + /* + * Compute the subset of (Fast) Read commands for which the 4-byte + * version is supported. + */ + discard_hwcaps = 0; + read_hwcaps = 0; + for (i = 0; i < ARRAY_SIZE(reads); i++) { + const struct sfdp_4bait *read = &reads[i]; + + discard_hwcaps |= read->hwcaps; + if ((params->hwcaps.mask & read->hwcaps) && + (dwords[0] & read->supported_bit)) + read_hwcaps |= read->hwcaps; + } + + /* + * Compute the subset of Page Program commands for which the 4-byte + * version is supported. + */ + pp_hwcaps = 0; + for (i = 0; i < ARRAY_SIZE(programs); i++) { + const struct sfdp_4bait *program = &programs[i]; + + /* + * The 4 Byte Address Instruction (Optional) Table is the only + * SFDP table that indicates support for Page Program Commands. + * Bypass the params->hwcaps.mask and consider 4BAIT the biggest + * authority for specifying Page Program support. + */ + discard_hwcaps |= program->hwcaps; + if (dwords[0] & program->supported_bit) + pp_hwcaps |= program->hwcaps; + } + + /* + * Compute the subset of Sector Erase commands for which the 4-byte + * version is supported. + */ + erase_mask = 0; + for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { + const struct sfdp_4bait *erase = &erases[i]; + + if (dwords[0] & erase->supported_bit) + erase_mask |= BIT(i); + } + + /* Replicate the sort done for the map's erase types in BFPT. */ + erase_mask = spi_nor_sort_erase_mask(map, erase_mask); + + /* + * We need at least one 4-byte op code per read, program and erase + * operation; the .read(), .write() and .erase() hooks share the + * nor->addr_width value. + */ + if (!read_hwcaps || !pp_hwcaps || !erase_mask) + goto out; + + /* + * Discard all operations from the 4-byte instruction set which are + * not supported by this memory. + */ + params->hwcaps.mask &= ~discard_hwcaps; + params->hwcaps.mask |= (read_hwcaps | pp_hwcaps); + + /* Use the 4-byte address instruction set. */ + for (i = 0; i < SNOR_CMD_READ_MAX; i++) { + struct spi_nor_read_command *read_cmd = ¶ms->reads[i]; + + read_cmd->opcode = spi_nor_convert_3to4_read(read_cmd->opcode); + } + + /* 4BAIT is the only SFDP table that indicates page program support. */ + if (pp_hwcaps & SNOR_HWCAPS_PP) + spi_nor_set_pp_settings(¶ms_pp[SNOR_CMD_PP], + SPINOR_OP_PP_4B, SNOR_PROTO_1_1_1); + if (pp_hwcaps & SNOR_HWCAPS_PP_1_1_4) + spi_nor_set_pp_settings(¶ms_pp[SNOR_CMD_PP_1_1_4], + SPINOR_OP_PP_1_1_4_4B, + SNOR_PROTO_1_1_4); + if (pp_hwcaps & SNOR_HWCAPS_PP_1_4_4) + spi_nor_set_pp_settings(¶ms_pp[SNOR_CMD_PP_1_4_4], + SPINOR_OP_PP_1_4_4_4B, + SNOR_PROTO_1_4_4); + + for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { + if (erase_mask & BIT(i)) + erase_type[i].opcode = (dwords[1] >> + erase_type[i].idx * 8) & 0xFF; + else + spi_nor_set_erase_type(&erase_type[i], 0u, 0xFF); + } + + /* + * We set SNOR_F_HAS_4BAIT in order to skip spi_nor_set_4byte_opcodes() + * later because we already did the conversion to 4byte opcodes. Also, + * this latest function implements a legacy quirk for the erase size of + * Spansion memory. However this quirk is no longer needed with new + * SFDP compliant memories. + */ + nor->addr_width = 4; + nor->flags |= SNOR_F_4B_OPCODES | SNOR_F_HAS_4BAIT; + + /* fall through */ +out: + kfree(dwords); + return ret; +} + +/** + * spi_nor_parse_sfdp() - parse the Serial Flash Discoverable Parameters. + * @nor: pointer to a 'struct spi_nor' + * @params: pointer to the 'struct spi_nor_flash_parameter' to be + * filled + * + * The Serial Flash Discoverable Parameters are described by the JEDEC JESD216 + * specification. This is a standard which tends to supported by almost all + * (Q)SPI memory manufacturers. Those hard-coded tables allow us to learn at + * runtime the main parameters needed to perform basic SPI flash operations such + * as Fast Read, Page Program or Sector Erase commands. + * + * Return: 0 on success, -errno otherwise. + */ +int spi_nor_parse_sfdp(struct spi_nor *nor, + struct spi_nor_flash_parameter *params) +{ + const struct sfdp_parameter_header *param_header, *bfpt_header; + struct sfdp_parameter_header *param_headers = NULL; + struct sfdp_header header; + struct device *dev = nor->dev; + size_t psize; + int i, err; + + /* Get the SFDP header. */ + err = spi_nor_read_sfdp_dma_unsafe(nor, 0, sizeof(header), &header); + if (err < 0) + return err; + + /* Check the SFDP header version. */ + if (le32_to_cpu(header.signature) != SFDP_SIGNATURE || + header.major != SFDP_JESD216_MAJOR) + return -EINVAL; + + /* + * Verify that the first and only mandatory parameter header is a + * Basic Flash Parameter Table header as specified in JESD216. + */ + bfpt_header = &header.bfpt_header; + if (SFDP_PARAM_HEADER_ID(bfpt_header) != SFDP_BFPT_ID || + bfpt_header->major != SFDP_JESD216_MAJOR) + return -EINVAL; + + /* + * Allocate memory then read all parameter headers with a single + * Read SFDP command. These parameter headers will actually be parsed + * twice: a first time to get the latest revision of the basic flash + * parameter table, then a second time to handle the supported optional + * tables. + * Hence we read the parameter headers once for all to reduce the + * processing time. Also we use kmalloc() instead of devm_kmalloc() + * because we don't need to keep these parameter headers: the allocated + * memory is always released with kfree() before exiting this function. + */ + if (header.nph) { + psize = header.nph * sizeof(*param_headers); + + param_headers = kmalloc(psize, GFP_KERNEL); + if (!param_headers) + return -ENOMEM; + + err = spi_nor_read_sfdp(nor, sizeof(header), + psize, param_headers); + if (err < 0) { + dev_dbg(dev, "failed to read SFDP parameter headers\n"); + goto exit; + } + } + + /* + * Check other parameter headers to get the latest revision of + * the basic flash parameter table. + */ + for (i = 0; i < header.nph; i++) { + param_header = ¶m_headers[i]; + + if (SFDP_PARAM_HEADER_ID(param_header) == SFDP_BFPT_ID && + param_header->major == SFDP_JESD216_MAJOR && + (param_header->minor > bfpt_header->minor || + (param_header->minor == bfpt_header->minor && + param_header->length > bfpt_header->length))) + bfpt_header = param_header; + } + + err = spi_nor_parse_bfpt(nor, bfpt_header, params); + if (err) + goto exit; + + /* Parse optional parameter tables. */ + for (i = 0; i < header.nph; i++) { + param_header = ¶m_headers[i]; + + switch (SFDP_PARAM_HEADER_ID(param_header)) { + case SFDP_SECTOR_MAP_ID: + err = spi_nor_parse_smpt(nor, param_header, params); + break; + + case SFDP_4BAIT_ID: + err = spi_nor_parse_4bait(nor, param_header, params); + break; + + default: + break; + } + + if (err) { + dev_warn(dev, "Failed to parse optional parameter table: %04x\n", + SFDP_PARAM_HEADER_ID(param_header)); + /* + * Let's not drop all information we extracted so far + * if optional table parsers fail. In case of failing, + * each optional parser is responsible to roll back to + * the previously known spi_nor data. + */ + err = 0; + } + } + +exit: + kfree(param_headers); + return err; +} diff --git a/drivers/mtd/spi-nor/sfdp.h b/drivers/mtd/spi-nor/sfdp.h new file mode 100644 index 000000000000..e0a8ded04890 --- /dev/null +++ b/drivers/mtd/spi-nor/sfdp.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#ifndef __LINUX_MTD_SFDP_H +#define __LINUX_MTD_SFDP_H + +/* Basic Flash Parameter Table */ + +/* + * JESD216 rev B defines a Basic Flash Parameter Table of 16 DWORDs. + * They are indexed from 1 but C arrays are indexed from 0. + */ +#define BFPT_DWORD(i) ((i) - 1) +#define BFPT_DWORD_MAX 16 + +struct sfdp_bfpt { + u32 dwords[BFPT_DWORD_MAX]; +}; + +/* The first version of JESD216 defined only 9 DWORDs. */ +#define BFPT_DWORD_MAX_JESD216 9 + +/* 1st DWORD. */ +#define BFPT_DWORD1_FAST_READ_1_1_2 BIT(16) +#define BFPT_DWORD1_ADDRESS_BYTES_MASK GENMASK(18, 17) +#define BFPT_DWORD1_ADDRESS_BYTES_3_ONLY (0x0UL << 17) +#define BFPT_DWORD1_ADDRESS_BYTES_3_OR_4 (0x1UL << 17) +#define BFPT_DWORD1_ADDRESS_BYTES_4_ONLY (0x2UL << 17) +#define BFPT_DWORD1_DTR BIT(19) +#define BFPT_DWORD1_FAST_READ_1_2_2 BIT(20) +#define BFPT_DWORD1_FAST_READ_1_4_4 BIT(21) +#define BFPT_DWORD1_FAST_READ_1_1_4 BIT(22) + +/* 5th DWORD. */ +#define BFPT_DWORD5_FAST_READ_2_2_2 BIT(0) +#define BFPT_DWORD5_FAST_READ_4_4_4 BIT(4) + +/* 11th DWORD. */ +#define BFPT_DWORD11_PAGE_SIZE_SHIFT 4 +#define BFPT_DWORD11_PAGE_SIZE_MASK GENMASK(7, 4) + +/* 15th DWORD. */ + +/* + * (from JESD216 rev B) + * Quad Enable Requirements (QER): + * - 000b: Device does not have a QE bit. Device detects 1-1-4 and 1-4-4 + * reads based on instruction. DQ3/HOLD# functions are hold during + * instruction phase. + * - 001b: QE is bit 1 of status register 2. It is set via Write Status with + * two data bytes where bit 1 of the second byte is one. + * [...] + * Writing only one byte to the status register has the side-effect of + * clearing status register 2, including the QE bit. The 100b code is + * used if writing one byte to the status register does not modify + * status register 2. + * - 010b: QE is bit 6 of status register 1. It is set via Write Status with + * one data byte where bit 6 is one. + * [...] + * - 011b: QE is bit 7 of status register 2. It is set via Write status + * register 2 instruction 3Eh with one data byte where bit 7 is one. + * [...] + * The status register 2 is read using instruction 3Fh. + * - 100b: QE is bit 1 of status register 2. It is set via Write Status with + * two data bytes where bit 1 of the second byte is one. + * [...] + * In contrast to the 001b code, writing one byte to the status + * register does not modify status register 2. + * - 101b: QE is bit 1 of status register 2. Status register 1 is read using + * Read Status instruction 05h. Status register2 is read using + * instruction 35h. QE is set via Write Status instruction 01h with + * two data bytes where bit 1 of the second byte is one. + * [...] + */ +#define BFPT_DWORD15_QER_MASK GENMASK(22, 20) +#define BFPT_DWORD15_QER_NONE (0x0UL << 20) /* Micron */ +#define BFPT_DWORD15_QER_SR2_BIT1_BUGGY (0x1UL << 20) +#define BFPT_DWORD15_QER_SR1_BIT6 (0x2UL << 20) /* Macronix */ +#define BFPT_DWORD15_QER_SR2_BIT7 (0x3UL << 20) +#define BFPT_DWORD15_QER_SR2_BIT1_NO_RD (0x4UL << 20) +#define BFPT_DWORD15_QER_SR2_BIT1 (0x5UL << 20) /* Spansion */ + +struct sfdp_parameter_header { + u8 id_lsb; + u8 minor; + u8 major; + u8 length; /* in double words */ + u8 parameter_table_pointer[3]; /* byte address */ + u8 id_msb; +}; + +int spi_nor_parse_sfdp(struct spi_nor *nor, + struct spi_nor_flash_parameter *params); + +#endif /* __LINUX_MTD_SFDP_H */ diff --git a/drivers/mtd/spi-nor/spansion.c b/drivers/mtd/spi-nor/spansion.c new file mode 100644 index 000000000000..6756202ace4b --- /dev/null +++ b/drivers/mtd/spi-nor/spansion.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info spansion_parts[] = { + /* Spansion/Cypress -- single (large) sector size only, at least + * for the chips listed here (without boot sectors). + */ + { "s25sl032p", INFO(0x010215, 0x4d00, 64 * 1024, 64, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "s25sl064p", INFO(0x010216, 0x4d00, 64 * 1024, 128, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "s25fl128s0", INFO6(0x012018, 0x4d0080, 256 * 1024, 64, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + USE_CLSR) }, + { "s25fl128s1", INFO6(0x012018, 0x4d0180, 64 * 1024, 256, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + USE_CLSR) }, + { "s25fl256s0", INFO(0x010219, 0x4d00, 256 * 1024, 128, USE_CLSR) }, + { "s25fl256s1", INFO(0x010219, 0x4d01, 64 * 1024, 512, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + USE_CLSR) }, + { "s25fl512s", INFO6(0x010220, 0x4d0080, 256 * 1024, 256, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | USE_CLSR) }, + { "s25fs512s", INFO6(0x010220, 0x4d0081, 256 * 1024, 256, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + USE_CLSR) }, + { "s70fl01gs", INFO(0x010221, 0x4d00, 256 * 1024, 256, 0) }, + { "s25sl12800", INFO(0x012018, 0x0300, 256 * 1024, 64, 0) }, + { "s25sl12801", INFO(0x012018, 0x0301, 64 * 1024, 256, 0) }, + { "s25fl129p0", INFO(0x012018, 0x4d00, 256 * 1024, 64, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + USE_CLSR) }, + { "s25fl129p1", INFO(0x012018, 0x4d01, 64 * 1024, 256, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + USE_CLSR) }, + { "s25sl004a", INFO(0x010212, 0, 64 * 1024, 8, 0) }, + { "s25sl008a", INFO(0x010213, 0, 64 * 1024, 16, 0) }, + { "s25sl016a", INFO(0x010214, 0, 64 * 1024, 32, 0) }, + { "s25sl032a", INFO(0x010215, 0, 64 * 1024, 64, 0) }, + { "s25sl064a", INFO(0x010216, 0, 64 * 1024, 128, 0) }, + { "s25fl004k", INFO(0xef4013, 0, 64 * 1024, 8, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "s25fl008k", INFO(0xef4014, 0, 64 * 1024, 16, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "s25fl016k", INFO(0xef4015, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "s25fl064k", INFO(0xef4017, 0, 64 * 1024, 128, SECT_4K) }, + { "s25fl116k", INFO(0x014015, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "s25fl132k", INFO(0x014016, 0, 64 * 1024, 64, SECT_4K) }, + { "s25fl164k", INFO(0x014017, 0, 64 * 1024, 128, SECT_4K) }, + { "s25fl204k", INFO(0x014013, 0, 64 * 1024, 8, + SECT_4K | SPI_NOR_DUAL_READ) }, + { "s25fl208k", INFO(0x014014, 0, 64 * 1024, 16, + SECT_4K | SPI_NOR_DUAL_READ) }, + { "s25fl064l", INFO(0x016017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_4B_OPCODES) }, + { "s25fl128l", INFO(0x016018, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_4B_OPCODES) }, + { "s25fl256l", INFO(0x016019, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_4B_OPCODES) }, +}; + +static void spansion_post_sfdp_fixups(struct spi_nor *nor) +{ + if (nor->params->size <= SZ_16M) + return; + + nor->flags |= SNOR_F_4B_OPCODES; + /* No small sector erase for 4-byte command set */ + nor->erase_opcode = SPINOR_OP_SE; + nor->mtd.erasesize = nor->info->sector_size; +} + +static const struct spi_nor_fixups spansion_fixups = { + .post_sfdp = spansion_post_sfdp_fixups, +}; + +const struct spi_nor_manufacturer spi_nor_spansion = { + .name = "spansion", + .parts = spansion_parts, + .nparts = ARRAY_SIZE(spansion_parts), + .fixups = &spansion_fixups, +}; diff --git a/drivers/mtd/spi-nor/spi-nor.c b/drivers/mtd/spi-nor/spi-nor.c deleted file mode 100644 index 4fc632ec18fe..000000000000 --- a/drivers/mtd/spi-nor/spi-nor.c +++ /dev/null @@ -1,5434 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Based on m25p80.c, by Mike Lavender (mike@steroidmicros.com), with - * influence from lart.c (Abraham Van Der Merwe) and mtd_dataflash.c - * - * Copyright (C) 2005, Intec Automation Inc. - * Copyright (C) 2014, Freescale Semiconductor, Inc. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -/* Define max times to check status register before we give up. */ - -/* - * For everything but full-chip erase; probably could be much smaller, but kept - * around for safety for now - */ -#define DEFAULT_READY_WAIT_JIFFIES (40UL * HZ) - -/* - * For full-chip erase, calibrated to a 2MB flash (M25P16); should be scaled up - * for larger flash - */ -#define CHIP_ERASE_2MB_READY_WAIT_JIFFIES (40UL * HZ) - -#define SPI_NOR_MAX_ID_LEN 6 -#define SPI_NOR_MAX_ADDR_WIDTH 4 - -struct sfdp_parameter_header { - u8 id_lsb; - u8 minor; - u8 major; - u8 length; /* in double words */ - u8 parameter_table_pointer[3]; /* byte address */ - u8 id_msb; -}; - -#define SFDP_PARAM_HEADER_ID(p) (((p)->id_msb << 8) | (p)->id_lsb) -#define SFDP_PARAM_HEADER_PTP(p) \ - (((p)->parameter_table_pointer[2] << 16) | \ - ((p)->parameter_table_pointer[1] << 8) | \ - ((p)->parameter_table_pointer[0] << 0)) - -#define SFDP_BFPT_ID 0xff00 /* Basic Flash Parameter Table */ -#define SFDP_SECTOR_MAP_ID 0xff81 /* Sector Map Table */ -#define SFDP_4BAIT_ID 0xff84 /* 4-byte Address Instruction Table */ - -#define SFDP_SIGNATURE 0x50444653U -#define SFDP_JESD216_MAJOR 1 -#define SFDP_JESD216_MINOR 0 -#define SFDP_JESD216A_MINOR 5 -#define SFDP_JESD216B_MINOR 6 - -struct sfdp_header { - u32 signature; /* Ox50444653U <=> "SFDP" */ - u8 minor; - u8 major; - u8 nph; /* 0-base number of parameter headers */ - u8 unused; - - /* Basic Flash Parameter Table. */ - struct sfdp_parameter_header bfpt_header; -}; - -/* Basic Flash Parameter Table */ - -/* - * JESD216 rev B defines a Basic Flash Parameter Table of 16 DWORDs. - * They are indexed from 1 but C arrays are indexed from 0. - */ -#define BFPT_DWORD(i) ((i) - 1) -#define BFPT_DWORD_MAX 16 - -/* The first version of JESD216 defined only 9 DWORDs. */ -#define BFPT_DWORD_MAX_JESD216 9 - -/* 1st DWORD. */ -#define BFPT_DWORD1_FAST_READ_1_1_2 BIT(16) -#define BFPT_DWORD1_ADDRESS_BYTES_MASK GENMASK(18, 17) -#define BFPT_DWORD1_ADDRESS_BYTES_3_ONLY (0x0UL << 17) -#define BFPT_DWORD1_ADDRESS_BYTES_3_OR_4 (0x1UL << 17) -#define BFPT_DWORD1_ADDRESS_BYTES_4_ONLY (0x2UL << 17) -#define BFPT_DWORD1_DTR BIT(19) -#define BFPT_DWORD1_FAST_READ_1_2_2 BIT(20) -#define BFPT_DWORD1_FAST_READ_1_4_4 BIT(21) -#define BFPT_DWORD1_FAST_READ_1_1_4 BIT(22) - -/* 5th DWORD. */ -#define BFPT_DWORD5_FAST_READ_2_2_2 BIT(0) -#define BFPT_DWORD5_FAST_READ_4_4_4 BIT(4) - -/* 11th DWORD. */ -#define BFPT_DWORD11_PAGE_SIZE_SHIFT 4 -#define BFPT_DWORD11_PAGE_SIZE_MASK GENMASK(7, 4) - -/* 15th DWORD. */ - -/* - * (from JESD216 rev B) - * Quad Enable Requirements (QER): - * - 000b: Device does not have a QE bit. Device detects 1-1-4 and 1-4-4 - * reads based on instruction. DQ3/HOLD# functions are hold during - * instruction phase. - * - 001b: QE is bit 1 of status register 2. It is set via Write Status with - * two data bytes where bit 1 of the second byte is one. - * [...] - * Writing only one byte to the status register has the side-effect of - * clearing status register 2, including the QE bit. The 100b code is - * used if writing one byte to the status register does not modify - * status register 2. - * - 010b: QE is bit 6 of status register 1. It is set via Write Status with - * one data byte where bit 6 is one. - * [...] - * - 011b: QE is bit 7 of status register 2. It is set via Write status - * register 2 instruction 3Eh with one data byte where bit 7 is one. - * [...] - * The status register 2 is read using instruction 3Fh. - * - 100b: QE is bit 1 of status register 2. It is set via Write Status with - * two data bytes where bit 1 of the second byte is one. - * [...] - * In contrast to the 001b code, writing one byte to the status - * register does not modify status register 2. - * - 101b: QE is bit 1 of status register 2. Status register 1 is read using - * Read Status instruction 05h. Status register2 is read using - * instruction 35h. QE is set via Write Status instruction 01h with - * two data bytes where bit 1 of the second byte is one. - * [...] - */ -#define BFPT_DWORD15_QER_MASK GENMASK(22, 20) -#define BFPT_DWORD15_QER_NONE (0x0UL << 20) /* Micron */ -#define BFPT_DWORD15_QER_SR2_BIT1_BUGGY (0x1UL << 20) -#define BFPT_DWORD15_QER_SR1_BIT6 (0x2UL << 20) /* Macronix */ -#define BFPT_DWORD15_QER_SR2_BIT7 (0x3UL << 20) -#define BFPT_DWORD15_QER_SR2_BIT1_NO_RD (0x4UL << 20) -#define BFPT_DWORD15_QER_SR2_BIT1 (0x5UL << 20) /* Spansion */ - -struct sfdp_bfpt { - u32 dwords[BFPT_DWORD_MAX]; -}; - -/** - * struct spi_nor_fixups - SPI NOR fixup hooks - * @default_init: called after default flash parameters init. Used to tweak - * flash parameters when information provided by the flash_info - * table is incomplete or wrong. - * @post_bfpt: called after the BFPT table has been parsed - * @post_sfdp: called after SFDP has been parsed (is also called for SPI NORs - * that do not support RDSFDP). Typically used to tweak various - * parameters that could not be extracted by other means (i.e. - * when information provided by the SFDP/flash_info tables are - * incomplete or wrong). - * - * Those hooks can be used to tweak the SPI NOR configuration when the SFDP - * table is broken or not available. - */ -struct spi_nor_fixups { - void (*default_init)(struct spi_nor *nor); - int (*post_bfpt)(struct spi_nor *nor, - const struct sfdp_parameter_header *bfpt_header, - const struct sfdp_bfpt *bfpt, - struct spi_nor_flash_parameter *params); - void (*post_sfdp)(struct spi_nor *nor); -}; - -struct flash_info { - char *name; - - /* - * This array stores the ID bytes. - * The first three bytes are the JEDIC ID. - * JEDEC ID zero means "no ID" (mostly older chips). - */ - u8 id[SPI_NOR_MAX_ID_LEN]; - u8 id_len; - - /* The size listed here is what works with SPINOR_OP_SE, which isn't - * necessarily called a "sector" by the vendor. - */ - unsigned sector_size; - u16 n_sectors; - - u16 page_size; - u16 addr_width; - - u32 flags; -#define SECT_4K BIT(0) /* SPINOR_OP_BE_4K works uniformly */ -#define SPI_NOR_NO_ERASE BIT(1) /* No erase command needed */ -#define SST_WRITE BIT(2) /* use SST byte programming */ -#define SPI_NOR_NO_FR BIT(3) /* Can't do fastread */ -#define SECT_4K_PMC BIT(4) /* SPINOR_OP_BE_4K_PMC works uniformly */ -#define SPI_NOR_DUAL_READ BIT(5) /* Flash supports Dual Read */ -#define SPI_NOR_QUAD_READ BIT(6) /* Flash supports Quad Read */ -#define USE_FSR BIT(7) /* use flag status register */ -#define SPI_NOR_HAS_LOCK BIT(8) /* Flash supports lock/unlock via SR */ -#define SPI_NOR_HAS_TB BIT(9) /* - * Flash SR has Top/Bottom (TB) protect - * bit. Must be used with - * SPI_NOR_HAS_LOCK. - */ -#define SPI_NOR_XSR_RDY BIT(10) /* - * S3AN flashes have specific opcode to - * read the status register. - * Flags SPI_NOR_XSR_RDY and SPI_S3AN - * use the same bit as one implies the - * other, but we will get rid of - * SPI_S3AN soon. - */ -#define SPI_S3AN BIT(10) /* - * Xilinx Spartan 3AN In-System Flash - * (MFR cannot be used for probing - * because it has the same value as - * ATMEL flashes) - */ -#define SPI_NOR_4B_OPCODES BIT(11) /* - * Use dedicated 4byte address op codes - * to support memory size above 128Mib. - */ -#define NO_CHIP_ERASE BIT(12) /* Chip does not support chip erase */ -#define SPI_NOR_SKIP_SFDP BIT(13) /* Skip parsing of SFDP tables */ -#define USE_CLSR BIT(14) /* use CLSR command */ -#define SPI_NOR_OCTAL_READ BIT(15) /* Flash supports Octal Read */ -#define SPI_NOR_TB_SR_BIT6 BIT(16) /* - * Top/Bottom (TB) is bit 6 of - * status register. Must be used with - * SPI_NOR_HAS_TB. - */ - - /* Part specific fixup hooks. */ - const struct spi_nor_fixups *fixups; -}; - -#define JEDEC_MFR(info) ((info)->id[0]) - -/** - * spi_nor_spimem_xfer_data() - helper function to read/write data to - * flash's memory region - * @nor: pointer to 'struct spi_nor' - * @op: pointer to 'struct spi_mem_op' template for transfer - * - * Return: number of bytes transferred on success, -errno otherwise - */ -static ssize_t spi_nor_spimem_xfer_data(struct spi_nor *nor, - struct spi_mem_op *op) -{ - bool usebouncebuf = false; - void *rdbuf = NULL; - const void *buf; - int ret; - - if (op->data.dir == SPI_MEM_DATA_IN) - buf = op->data.buf.in; - else - buf = op->data.buf.out; - - if (object_is_on_stack(buf) || !virt_addr_valid(buf)) - usebouncebuf = true; - - if (usebouncebuf) { - if (op->data.nbytes > nor->bouncebuf_size) - op->data.nbytes = nor->bouncebuf_size; - - if (op->data.dir == SPI_MEM_DATA_IN) { - rdbuf = op->data.buf.in; - op->data.buf.in = nor->bouncebuf; - } else { - op->data.buf.out = nor->bouncebuf; - memcpy(nor->bouncebuf, buf, - op->data.nbytes); - } - } - - ret = spi_mem_adjust_op_size(nor->spimem, op); - if (ret) - return ret; - - ret = spi_mem_exec_op(nor->spimem, op); - if (ret) - return ret; - - if (usebouncebuf && op->data.dir == SPI_MEM_DATA_IN) - memcpy(rdbuf, nor->bouncebuf, op->data.nbytes); - - return op->data.nbytes; -} - -/** - * spi_nor_spimem_read_data() - read data from flash's memory region via - * spi-mem - * @nor: pointer to 'struct spi_nor' - * @from: offset to read from - * @len: number of bytes to read - * @buf: pointer to dst buffer - * - * Return: number of bytes read successfully, -errno otherwise - */ -static ssize_t spi_nor_spimem_read_data(struct spi_nor *nor, loff_t from, - size_t len, u8 *buf) -{ - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(nor->read_opcode, 1), - SPI_MEM_OP_ADDR(nor->addr_width, from, 1), - SPI_MEM_OP_DUMMY(nor->read_dummy, 1), - SPI_MEM_OP_DATA_IN(len, buf, 1)); - - /* get transfer protocols. */ - op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(nor->read_proto); - op.addr.buswidth = spi_nor_get_protocol_addr_nbits(nor->read_proto); - op.dummy.buswidth = op.addr.buswidth; - op.data.buswidth = spi_nor_get_protocol_data_nbits(nor->read_proto); - - /* convert the dummy cycles to the number of bytes */ - op.dummy.nbytes = (nor->read_dummy * op.dummy.buswidth) / 8; - - return spi_nor_spimem_xfer_data(nor, &op); -} - -/** - * spi_nor_read_data() - read data from flash memory - * @nor: pointer to 'struct spi_nor' - * @from: offset to read from - * @len: number of bytes to read - * @buf: pointer to dst buffer - * - * Return: number of bytes read successfully, -errno otherwise - */ -static ssize_t spi_nor_read_data(struct spi_nor *nor, loff_t from, size_t len, - u8 *buf) -{ - if (nor->spimem) - return spi_nor_spimem_read_data(nor, from, len, buf); - - return nor->controller_ops->read(nor, from, len, buf); -} - -/** - * spi_nor_spimem_write_data() - write data to flash memory via - * spi-mem - * @nor: pointer to 'struct spi_nor' - * @to: offset to write to - * @len: number of bytes to write - * @buf: pointer to src buffer - * - * Return: number of bytes written successfully, -errno otherwise - */ -static ssize_t spi_nor_spimem_write_data(struct spi_nor *nor, loff_t to, - size_t len, const u8 *buf) -{ - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(nor->program_opcode, 1), - SPI_MEM_OP_ADDR(nor->addr_width, to, 1), - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_OUT(len, buf, 1)); - - op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(nor->write_proto); - op.addr.buswidth = spi_nor_get_protocol_addr_nbits(nor->write_proto); - op.data.buswidth = spi_nor_get_protocol_data_nbits(nor->write_proto); - - if (nor->program_opcode == SPINOR_OP_AAI_WP && nor->sst_write_second) - op.addr.nbytes = 0; - - return spi_nor_spimem_xfer_data(nor, &op); -} - -/** - * spi_nor_write_data() - write data to flash memory - * @nor: pointer to 'struct spi_nor' - * @to: offset to write to - * @len: number of bytes to write - * @buf: pointer to src buffer - * - * Return: number of bytes written successfully, -errno otherwise - */ -static ssize_t spi_nor_write_data(struct spi_nor *nor, loff_t to, size_t len, - const u8 *buf) -{ - if (nor->spimem) - return spi_nor_spimem_write_data(nor, to, len, buf); - - return nor->controller_ops->write(nor, to, len, buf); -} - -/** - * spi_nor_write_enable() - Set write enable latch with Write Enable command. - * @nor: pointer to 'struct spi_nor'. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_write_enable(struct spi_nor *nor) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WREN, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_NO_DATA); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WREN, - NULL, 0); - } - - if (ret) - dev_dbg(nor->dev, "error %d on Write Enable\n", ret); - - return ret; -} - -/** - * spi_nor_write_disable() - Send Write Disable instruction to the chip. - * @nor: pointer to 'struct spi_nor'. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_write_disable(struct spi_nor *nor) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WRDI, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_NO_DATA); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WRDI, - NULL, 0); - } - - if (ret) - dev_dbg(nor->dev, "error %d on Write Disable\n", ret); - - return ret; -} - -/** - * spi_nor_read_sr() - Read the Status Register. - * @nor: pointer to 'struct spi_nor'. - * @sr: pointer to a DMA-able buffer where the value of the - * Status Register will be written. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_read_sr(struct spi_nor *nor, u8 *sr) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDSR, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_IN(1, sr, 1)); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->read_reg(nor, SPINOR_OP_RDSR, - sr, 1); - } - - if (ret) - dev_dbg(nor->dev, "error %d reading SR\n", ret); - - return ret; -} - -/** - * spi_nor_read_fsr() - Read the Flag Status Register. - * @nor: pointer to 'struct spi_nor' - * @fsr: pointer to a DMA-able buffer where the value of the - * Flag Status Register will be written. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_read_fsr(struct spi_nor *nor, u8 *fsr) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDFSR, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_IN(1, fsr, 1)); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->read_reg(nor, SPINOR_OP_RDFSR, - fsr, 1); - } - - if (ret) - dev_dbg(nor->dev, "error %d reading FSR\n", ret); - - return ret; -} - -/** - * spi_nor_read_cr() - Read the Configuration Register using the - * SPINOR_OP_RDCR (35h) command. - * @nor: pointer to 'struct spi_nor' - * @cr: pointer to a DMA-able buffer where the value of the - * Configuration Register will be written. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_read_cr(struct spi_nor *nor, u8 *cr) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDCR, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_IN(1, cr, 1)); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->read_reg(nor, SPINOR_OP_RDCR, cr, 1); - } - - if (ret) - dev_dbg(nor->dev, "error %d reading CR\n", ret); - - return ret; -} - -/** - * macronix_set_4byte() - Set 4-byte address mode for Macronix flashes. - * @nor: pointer to 'struct spi_nor'. - * @enable: true to enter the 4-byte address mode, false to exit the 4-byte - * address mode. - * - * Return: 0 on success, -errno otherwise. - */ -static int macronix_set_4byte(struct spi_nor *nor, bool enable) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(enable ? - SPINOR_OP_EN4B : - SPINOR_OP_EX4B, - 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_NO_DATA); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, - enable ? SPINOR_OP_EN4B : - SPINOR_OP_EX4B, - NULL, 0); - } - - if (ret) - dev_dbg(nor->dev, "error %d setting 4-byte mode\n", ret); - - return ret; -} - -/** - * st_micron_set_4byte() - Set 4-byte address mode for ST and Micron flashes. - * @nor: pointer to 'struct spi_nor'. - * @enable: true to enter the 4-byte address mode, false to exit the 4-byte - * address mode. - * - * Return: 0 on success, -errno otherwise. - */ -static int st_micron_set_4byte(struct spi_nor *nor, bool enable) -{ - int ret; - - ret = spi_nor_write_enable(nor); - if (ret) - return ret; - - ret = macronix_set_4byte(nor, enable); - if (ret) - return ret; - - return spi_nor_write_disable(nor); -} - -/** - * spansion_set_4byte() - Set 4-byte address mode for Spansion flashes. - * @nor: pointer to 'struct spi_nor'. - * @enable: true to enter the 4-byte address mode, false to exit the 4-byte - * address mode. - * - * Return: 0 on success, -errno otherwise. - */ -static int spansion_set_4byte(struct spi_nor *nor, bool enable) -{ - int ret; - - nor->bouncebuf[0] = enable << 7; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_BRWR, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_OUT(1, nor->bouncebuf, 1)); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, SPINOR_OP_BRWR, - nor->bouncebuf, 1); - } - - if (ret) - dev_dbg(nor->dev, "error %d setting 4-byte mode\n", ret); - - return ret; -} - -/** - * spi_nor_write_ear() - Write Extended Address Register. - * @nor: pointer to 'struct spi_nor'. - * @ear: value to write to the Extended Address Register. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_write_ear(struct spi_nor *nor, u8 ear) -{ - int ret; - - nor->bouncebuf[0] = ear; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WREAR, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_OUT(1, nor->bouncebuf, 1)); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WREAR, - nor->bouncebuf, 1); - } - - if (ret) - dev_dbg(nor->dev, "error %d writing EAR\n", ret); - - return ret; -} - -/** - * winbond_set_4byte() - Set 4-byte address mode for Winbond flashes. - * @nor: pointer to 'struct spi_nor'. - * @enable: true to enter the 4-byte address mode, false to exit the 4-byte - * address mode. - * - * Return: 0 on success, -errno otherwise. - */ -static int winbond_set_4byte(struct spi_nor *nor, bool enable) -{ - int ret; - - ret = macronix_set_4byte(nor, enable); - if (ret || enable) - return ret; - - /* - * On Winbond W25Q256FV, leaving 4byte mode causes the Extended Address - * Register to be set to 1, so all 3-byte-address reads come from the - * second 16M. We must clear the register to enable normal behavior. - */ - ret = spi_nor_write_enable(nor); - if (ret) - return ret; - - ret = spi_nor_write_ear(nor, 0); - if (ret) - return ret; - - return spi_nor_write_disable(nor); -} - -/** - * spi_nor_xread_sr() - Read the Status Register on S3AN flashes. - * @nor: pointer to 'struct spi_nor'. - * @sr: pointer to a DMA-able buffer where the value of the - * Status Register will be written. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_xread_sr(struct spi_nor *nor, u8 *sr) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_XRDSR, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_IN(1, sr, 1)); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->read_reg(nor, SPINOR_OP_XRDSR, - sr, 1); - } - - if (ret) - dev_dbg(nor->dev, "error %d reading XRDSR\n", ret); - - return ret; -} - -/** - * s3an_sr_ready() - Query the Status Register of the S3AN flash to see if the - * flash is ready for new commands. - * @nor: pointer to 'struct spi_nor'. - * - * Return: 0 on success, -errno otherwise. - */ -static int s3an_sr_ready(struct spi_nor *nor) -{ - int ret; - - ret = spi_nor_xread_sr(nor, nor->bouncebuf); - if (ret) - return ret; - - return !!(nor->bouncebuf[0] & XSR_RDY); -} - -/** - * spi_nor_clear_sr() - Clear the Status Register. - * @nor: pointer to 'struct spi_nor'. - */ -static void spi_nor_clear_sr(struct spi_nor *nor) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_CLSR, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_NO_DATA); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, SPINOR_OP_CLSR, - NULL, 0); - } - - if (ret) - dev_dbg(nor->dev, "error %d clearing SR\n", ret); -} - -/** - * spi_nor_sr_ready() - Query the Status Register to see if the flash is ready - * for new commands. - * @nor: pointer to 'struct spi_nor'. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_sr_ready(struct spi_nor *nor) -{ - int ret = spi_nor_read_sr(nor, nor->bouncebuf); - - if (ret) - return ret; - - if (nor->flags & SNOR_F_USE_CLSR && - nor->bouncebuf[0] & (SR_E_ERR | SR_P_ERR)) { - if (nor->bouncebuf[0] & SR_E_ERR) - dev_err(nor->dev, "Erase Error occurred\n"); - else - dev_err(nor->dev, "Programming Error occurred\n"); - - spi_nor_clear_sr(nor); - return -EIO; - } - - return !(nor->bouncebuf[0] & SR_WIP); -} - -/** - * spi_nor_clear_fsr() - Clear the Flag Status Register. - * @nor: pointer to 'struct spi_nor'. - */ -static void spi_nor_clear_fsr(struct spi_nor *nor) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_CLFSR, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_NO_DATA); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, SPINOR_OP_CLFSR, - NULL, 0); - } - - if (ret) - dev_dbg(nor->dev, "error %d clearing FSR\n", ret); -} - -/** - * spi_nor_fsr_ready() - Query the Flag Status Register to see if the flash is - * ready for new commands. - * @nor: pointer to 'struct spi_nor'. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_fsr_ready(struct spi_nor *nor) -{ - int ret = spi_nor_read_fsr(nor, nor->bouncebuf); - - if (ret) - return ret; - - if (nor->bouncebuf[0] & (FSR_E_ERR | FSR_P_ERR)) { - if (nor->bouncebuf[0] & FSR_E_ERR) - dev_err(nor->dev, "Erase operation failed.\n"); - else - dev_err(nor->dev, "Program operation failed.\n"); - - if (nor->bouncebuf[0] & FSR_PT_ERR) - dev_err(nor->dev, - "Attempted to modify a protected sector.\n"); - - spi_nor_clear_fsr(nor); - return -EIO; - } - - return nor->bouncebuf[0] & FSR_READY; -} - -/** - * spi_nor_ready() - Query the flash to see if it is ready for new commands. - * @nor: pointer to 'struct spi_nor'. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_ready(struct spi_nor *nor) -{ - int sr, fsr; - - if (nor->flags & SNOR_F_READY_XSR_RDY) - sr = s3an_sr_ready(nor); - else - sr = spi_nor_sr_ready(nor); - if (sr < 0) - return sr; - fsr = nor->flags & SNOR_F_USE_FSR ? spi_nor_fsr_ready(nor) : 1; - if (fsr < 0) - return fsr; - return sr && fsr; -} - -/** - * spi_nor_wait_till_ready_with_timeout() - Service routine to read the - * Status Register until ready, or timeout occurs. - * @nor: pointer to "struct spi_nor". - * @timeout_jiffies: jiffies to wait until timeout. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_wait_till_ready_with_timeout(struct spi_nor *nor, - unsigned long timeout_jiffies) -{ - unsigned long deadline; - int timeout = 0, ret; - - deadline = jiffies + timeout_jiffies; - - while (!timeout) { - if (time_after_eq(jiffies, deadline)) - timeout = 1; - - ret = spi_nor_ready(nor); - if (ret < 0) - return ret; - if (ret) - return 0; - - cond_resched(); - } - - dev_dbg(nor->dev, "flash operation timed out\n"); - - return -ETIMEDOUT; -} - -/** - * spi_nor_wait_till_ready() - Wait for a predefined amount of time for the - * flash to be ready, or timeout occurs. - * @nor: pointer to "struct spi_nor". - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_wait_till_ready(struct spi_nor *nor) -{ - return spi_nor_wait_till_ready_with_timeout(nor, - DEFAULT_READY_WAIT_JIFFIES); -} - -/** - * spi_nor_write_sr() - Write the Status Register. - * @nor: pointer to 'struct spi_nor'. - * @sr: pointer to DMA-able buffer to write to the Status Register. - * @len: number of bytes to write to the Status Register. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_write_sr(struct spi_nor *nor, const u8 *sr, size_t len) -{ - int ret; - - ret = spi_nor_write_enable(nor); - if (ret) - return ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WRSR, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_OUT(len, sr, 1)); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WRSR, - sr, len); - } - - if (ret) { - dev_dbg(nor->dev, "error %d writing SR\n", ret); - return ret; - } - - return spi_nor_wait_till_ready(nor); -} - -/** - * spi_nor_write_sr1_and_check() - Write one byte to the Status Register 1 and - * ensure that the byte written match the received value. - * @nor: pointer to a 'struct spi_nor'. - * @sr1: byte value to be written to the Status Register. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_write_sr1_and_check(struct spi_nor *nor, u8 sr1) -{ - int ret; - - nor->bouncebuf[0] = sr1; - - ret = spi_nor_write_sr(nor, nor->bouncebuf, 1); - if (ret) - return ret; - - ret = spi_nor_read_sr(nor, nor->bouncebuf); - if (ret) - return ret; - - if (nor->bouncebuf[0] != sr1) { - dev_dbg(nor->dev, "SR1: read back test failed\n"); - return -EIO; - } - - return 0; -} - -/** - * spi_nor_write_16bit_sr_and_check() - Write the Status Register 1 and the - * Status Register 2 in one shot. Ensure that the byte written in the Status - * Register 1 match the received value, and that the 16-bit Write did not - * affect what was already in the Status Register 2. - * @nor: pointer to a 'struct spi_nor'. - * @sr1: byte value to be written to the Status Register 1. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_write_16bit_sr_and_check(struct spi_nor *nor, u8 sr1) -{ - int ret; - u8 *sr_cr = nor->bouncebuf; - u8 cr_written; - - /* Make sure we don't overwrite the contents of Status Register 2. */ - if (!(nor->flags & SNOR_F_NO_READ_CR)) { - ret = spi_nor_read_cr(nor, &sr_cr[1]); - if (ret) - return ret; - } else if (nor->params.quad_enable) { - /* - * If the Status Register 2 Read command (35h) is not - * supported, we should at least be sure we don't - * change the value of the SR2 Quad Enable bit. - * - * We can safely assume that when the Quad Enable method is - * set, the value of the QE bit is one, as a consequence of the - * nor->params.quad_enable() call. - * - * We can safely assume that the Quad Enable bit is present in - * the Status Register 2 at BIT(1). According to the JESD216 - * revB standard, BFPT DWORDS[15], bits 22:20, the 16-bit - * Write Status (01h) command is available just for the cases - * in which the QE bit is described in SR2 at BIT(1). - */ - sr_cr[1] = SR2_QUAD_EN_BIT1; - } else { - sr_cr[1] = 0; - } - - sr_cr[0] = sr1; - - ret = spi_nor_write_sr(nor, sr_cr, 2); - if (ret) - return ret; - - if (nor->flags & SNOR_F_NO_READ_CR) - return 0; - - cr_written = sr_cr[1]; - - ret = spi_nor_read_cr(nor, &sr_cr[1]); - if (ret) - return ret; - - if (cr_written != sr_cr[1]) { - dev_dbg(nor->dev, "CR: read back test failed\n"); - return -EIO; - } - - return 0; -} - -/** - * spi_nor_write_16bit_cr_and_check() - Write the Status Register 1 and the - * Configuration Register in one shot. Ensure that the byte written in the - * Configuration Register match the received value, and that the 16-bit Write - * did not affect what was already in the Status Register 1. - * @nor: pointer to a 'struct spi_nor'. - * @cr: byte value to be written to the Configuration Register. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_write_16bit_cr_and_check(struct spi_nor *nor, u8 cr) -{ - int ret; - u8 *sr_cr = nor->bouncebuf; - u8 sr_written; - - /* Keep the current value of the Status Register 1. */ - ret = spi_nor_read_sr(nor, sr_cr); - if (ret) - return ret; - - sr_cr[1] = cr; - - ret = spi_nor_write_sr(nor, sr_cr, 2); - if (ret) - return ret; - - sr_written = sr_cr[0]; - - ret = spi_nor_read_sr(nor, sr_cr); - if (ret) - return ret; - - if (sr_written != sr_cr[0]) { - dev_dbg(nor->dev, "SR: Read back test failed\n"); - return -EIO; - } - - if (nor->flags & SNOR_F_NO_READ_CR) - return 0; - - ret = spi_nor_read_cr(nor, &sr_cr[1]); - if (ret) - return ret; - - if (cr != sr_cr[1]) { - dev_dbg(nor->dev, "CR: read back test failed\n"); - return -EIO; - } - - return 0; -} - -/** - * spi_nor_write_sr_and_check() - Write the Status Register 1 and ensure that - * the byte written match the received value without affecting other bits in the - * Status Register 1 and 2. - * @nor: pointer to a 'struct spi_nor'. - * @sr1: byte value to be written to the Status Register. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_write_sr_and_check(struct spi_nor *nor, u8 sr1) -{ - if (nor->flags & SNOR_F_HAS_16BIT_SR) - return spi_nor_write_16bit_sr_and_check(nor, sr1); - - return spi_nor_write_sr1_and_check(nor, sr1); -} - -/** - * spi_nor_write_sr2() - Write the Status Register 2 using the - * SPINOR_OP_WRSR2 (3eh) command. - * @nor: pointer to 'struct spi_nor'. - * @sr2: pointer to DMA-able buffer to write to the Status Register 2. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_write_sr2(struct spi_nor *nor, const u8 *sr2) -{ - int ret; - - ret = spi_nor_write_enable(nor); - if (ret) - return ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WRSR2, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_OUT(1, sr2, 1)); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, SPINOR_OP_WRSR2, - sr2, 1); - } - - if (ret) { - dev_dbg(nor->dev, "error %d writing SR2\n", ret); - return ret; - } - - return spi_nor_wait_till_ready(nor); -} - -/** - * spi_nor_read_sr2() - Read the Status Register 2 using the - * SPINOR_OP_RDSR2 (3fh) command. - * @nor: pointer to 'struct spi_nor'. - * @sr2: pointer to DMA-able buffer where the value of the - * Status Register 2 will be written. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_read_sr2(struct spi_nor *nor, u8 *sr2) -{ - int ret; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDSR2, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_IN(1, sr2, 1)); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->read_reg(nor, SPINOR_OP_RDSR2, - sr2, 1); - } - - if (ret) - dev_dbg(nor->dev, "error %d reading SR2\n", ret); - - return ret; -} - -/** - * spi_nor_erase_chip() - Erase the entire flash memory. - * @nor: pointer to 'struct spi_nor'. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_erase_chip(struct spi_nor *nor) -{ - int ret; - - dev_dbg(nor->dev, " %lldKiB\n", (long long)(nor->mtd.size >> 10)); - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_CHIP_ERASE, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_NO_DATA); - - ret = spi_mem_exec_op(nor->spimem, &op); - } else { - ret = nor->controller_ops->write_reg(nor, SPINOR_OP_CHIP_ERASE, - NULL, 0); - } - - if (ret) - dev_dbg(nor->dev, "error %d erasing chip\n", ret); - - return ret; -} - -static struct spi_nor *mtd_to_spi_nor(struct mtd_info *mtd) -{ - return mtd->priv; -} - -static u8 spi_nor_convert_opcode(u8 opcode, const u8 table[][2], size_t size) -{ - size_t i; - - for (i = 0; i < size; i++) - if (table[i][0] == opcode) - return table[i][1]; - - /* No conversion found, keep input op code. */ - return opcode; -} - -static u8 spi_nor_convert_3to4_read(u8 opcode) -{ - static const u8 spi_nor_3to4_read[][2] = { - { SPINOR_OP_READ, SPINOR_OP_READ_4B }, - { SPINOR_OP_READ_FAST, SPINOR_OP_READ_FAST_4B }, - { SPINOR_OP_READ_1_1_2, SPINOR_OP_READ_1_1_2_4B }, - { SPINOR_OP_READ_1_2_2, SPINOR_OP_READ_1_2_2_4B }, - { SPINOR_OP_READ_1_1_4, SPINOR_OP_READ_1_1_4_4B }, - { SPINOR_OP_READ_1_4_4, SPINOR_OP_READ_1_4_4_4B }, - { SPINOR_OP_READ_1_1_8, SPINOR_OP_READ_1_1_8_4B }, - { SPINOR_OP_READ_1_8_8, SPINOR_OP_READ_1_8_8_4B }, - - { SPINOR_OP_READ_1_1_1_DTR, SPINOR_OP_READ_1_1_1_DTR_4B }, - { SPINOR_OP_READ_1_2_2_DTR, SPINOR_OP_READ_1_2_2_DTR_4B }, - { SPINOR_OP_READ_1_4_4_DTR, SPINOR_OP_READ_1_4_4_DTR_4B }, - }; - - return spi_nor_convert_opcode(opcode, spi_nor_3to4_read, - ARRAY_SIZE(spi_nor_3to4_read)); -} - -static u8 spi_nor_convert_3to4_program(u8 opcode) -{ - static const u8 spi_nor_3to4_program[][2] = { - { SPINOR_OP_PP, SPINOR_OP_PP_4B }, - { SPINOR_OP_PP_1_1_4, SPINOR_OP_PP_1_1_4_4B }, - { SPINOR_OP_PP_1_4_4, SPINOR_OP_PP_1_4_4_4B }, - { SPINOR_OP_PP_1_1_8, SPINOR_OP_PP_1_1_8_4B }, - { SPINOR_OP_PP_1_8_8, SPINOR_OP_PP_1_8_8_4B }, - }; - - return spi_nor_convert_opcode(opcode, spi_nor_3to4_program, - ARRAY_SIZE(spi_nor_3to4_program)); -} - -static u8 spi_nor_convert_3to4_erase(u8 opcode) -{ - static const u8 spi_nor_3to4_erase[][2] = { - { SPINOR_OP_BE_4K, SPINOR_OP_BE_4K_4B }, - { SPINOR_OP_BE_32K, SPINOR_OP_BE_32K_4B }, - { SPINOR_OP_SE, SPINOR_OP_SE_4B }, - }; - - return spi_nor_convert_opcode(opcode, spi_nor_3to4_erase, - ARRAY_SIZE(spi_nor_3to4_erase)); -} - -static void spi_nor_set_4byte_opcodes(struct spi_nor *nor) -{ - nor->read_opcode = spi_nor_convert_3to4_read(nor->read_opcode); - nor->program_opcode = spi_nor_convert_3to4_program(nor->program_opcode); - nor->erase_opcode = spi_nor_convert_3to4_erase(nor->erase_opcode); - - if (!spi_nor_has_uniform_erase(nor)) { - struct spi_nor_erase_map *map = &nor->params.erase_map; - struct spi_nor_erase_type *erase; - int i; - - for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { - erase = &map->erase_type[i]; - erase->opcode = - spi_nor_convert_3to4_erase(erase->opcode); - } - } -} - -static int spi_nor_lock_and_prep(struct spi_nor *nor) -{ - int ret = 0; - - mutex_lock(&nor->lock); - - if (nor->controller_ops && nor->controller_ops->prepare) { - ret = nor->controller_ops->prepare(nor); - if (ret) { - mutex_unlock(&nor->lock); - return ret; - } - } - return ret; -} - -static void spi_nor_unlock_and_unprep(struct spi_nor *nor) -{ - if (nor->controller_ops && nor->controller_ops->unprepare) - nor->controller_ops->unprepare(nor); - mutex_unlock(&nor->lock); -} - -/* - * This code converts an address to the Default Address Mode, that has non - * power of two page sizes. We must support this mode because it is the default - * mode supported by Xilinx tools, it can access the whole flash area and - * changing over to the Power-of-two mode is irreversible and corrupts the - * original data. - * Addr can safely be unsigned int, the biggest S3AN device is smaller than - * 4 MiB. - */ -static u32 s3an_convert_addr(struct spi_nor *nor, u32 addr) -{ - u32 offset, page; - - offset = addr % nor->page_size; - page = addr / nor->page_size; - page <<= (nor->page_size > 512) ? 10 : 9; - - return page | offset; -} - -static u32 spi_nor_convert_addr(struct spi_nor *nor, loff_t addr) -{ - if (!nor->params.convert_addr) - return addr; - - return nor->params.convert_addr(nor, addr); -} - -/* - * Initiate the erasure of a single sector - */ -static int spi_nor_erase_sector(struct spi_nor *nor, u32 addr) -{ - int i; - - addr = spi_nor_convert_addr(nor, addr); - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(nor->erase_opcode, 1), - SPI_MEM_OP_ADDR(nor->addr_width, addr, 1), - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_NO_DATA); - - return spi_mem_exec_op(nor->spimem, &op); - } else if (nor->controller_ops->erase) { - return nor->controller_ops->erase(nor, addr); - } - - /* - * Default implementation, if driver doesn't have a specialized HW - * control - */ - for (i = nor->addr_width - 1; i >= 0; i--) { - nor->bouncebuf[i] = addr & 0xff; - addr >>= 8; - } - - return nor->controller_ops->write_reg(nor, nor->erase_opcode, - nor->bouncebuf, nor->addr_width); -} - -/** - * spi_nor_div_by_erase_size() - calculate remainder and update new dividend - * @erase: pointer to a structure that describes a SPI NOR erase type - * @dividend: dividend value - * @remainder: pointer to u32 remainder (will be updated) - * - * Return: the result of the division - */ -static u64 spi_nor_div_by_erase_size(const struct spi_nor_erase_type *erase, - u64 dividend, u32 *remainder) -{ - /* JEDEC JESD216B Standard imposes erase sizes to be power of 2. */ - *remainder = (u32)dividend & erase->size_mask; - return dividend >> erase->size_shift; -} - -/** - * spi_nor_find_best_erase_type() - find the best erase type for the given - * offset in the serial flash memory and the - * number of bytes to erase. The region in - * which the address fits is expected to be - * provided. - * @map: the erase map of the SPI NOR - * @region: pointer to a structure that describes a SPI NOR erase region - * @addr: offset in the serial flash memory - * @len: number of bytes to erase - * - * Return: a pointer to the best fitted erase type, NULL otherwise. - */ -static const struct spi_nor_erase_type * -spi_nor_find_best_erase_type(const struct spi_nor_erase_map *map, - const struct spi_nor_erase_region *region, - u64 addr, u32 len) -{ - const struct spi_nor_erase_type *erase; - u32 rem; - int i; - u8 erase_mask = region->offset & SNOR_ERASE_TYPE_MASK; - - /* - * Erase types are ordered by size, with the smallest erase type at - * index 0. - */ - for (i = SNOR_ERASE_TYPE_MAX - 1; i >= 0; i--) { - /* Does the erase region support the tested erase type? */ - if (!(erase_mask & BIT(i))) - continue; - - erase = &map->erase_type[i]; - - /* Don't erase more than what the user has asked for. */ - if (erase->size > len) - continue; - - /* Alignment is not mandatory for overlaid regions */ - if (region->offset & SNOR_OVERLAID_REGION) - return erase; - - spi_nor_div_by_erase_size(erase, addr, &rem); - if (rem) - continue; - else - return erase; - } - - return NULL; -} - -/** - * spi_nor_region_next() - get the next spi nor region - * @region: pointer to a structure that describes a SPI NOR erase region - * - * Return: the next spi nor region or NULL if last region. - */ -static struct spi_nor_erase_region * -spi_nor_region_next(struct spi_nor_erase_region *region) -{ - if (spi_nor_region_is_last(region)) - return NULL; - region++; - return region; -} - -/** - * spi_nor_find_erase_region() - find the region of the serial flash memory in - * which the offset fits - * @map: the erase map of the SPI NOR - * @addr: offset in the serial flash memory - * - * Return: a pointer to the spi_nor_erase_region struct, ERR_PTR(-errno) - * otherwise. - */ -static struct spi_nor_erase_region * -spi_nor_find_erase_region(const struct spi_nor_erase_map *map, u64 addr) -{ - struct spi_nor_erase_region *region = map->regions; - u64 region_start = region->offset & ~SNOR_ERASE_FLAGS_MASK; - u64 region_end = region_start + region->size; - - while (addr < region_start || addr >= region_end) { - region = spi_nor_region_next(region); - if (!region) - return ERR_PTR(-EINVAL); - - region_start = region->offset & ~SNOR_ERASE_FLAGS_MASK; - region_end = region_start + region->size; - } - - return region; -} - -/** - * spi_nor_init_erase_cmd() - initialize an erase command - * @region: pointer to a structure that describes a SPI NOR erase region - * @erase: pointer to a structure that describes a SPI NOR erase type - * - * Return: the pointer to the allocated erase command, ERR_PTR(-errno) - * otherwise. - */ -static struct spi_nor_erase_command * -spi_nor_init_erase_cmd(const struct spi_nor_erase_region *region, - const struct spi_nor_erase_type *erase) -{ - struct spi_nor_erase_command *cmd; - - cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); - if (!cmd) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD(&cmd->list); - cmd->opcode = erase->opcode; - cmd->count = 1; - - if (region->offset & SNOR_OVERLAID_REGION) - cmd->size = region->size; - else - cmd->size = erase->size; - - return cmd; -} - -/** - * spi_nor_destroy_erase_cmd_list() - destroy erase command list - * @erase_list: list of erase commands - */ -static void spi_nor_destroy_erase_cmd_list(struct list_head *erase_list) -{ - struct spi_nor_erase_command *cmd, *next; - - list_for_each_entry_safe(cmd, next, erase_list, list) { - list_del(&cmd->list); - kfree(cmd); - } -} - -/** - * spi_nor_init_erase_cmd_list() - initialize erase command list - * @nor: pointer to a 'struct spi_nor' - * @erase_list: list of erase commands to be executed once we validate that the - * erase can be performed - * @addr: offset in the serial flash memory - * @len: number of bytes to erase - * - * Builds the list of best fitted erase commands and verifies if the erase can - * be performed. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_init_erase_cmd_list(struct spi_nor *nor, - struct list_head *erase_list, - u64 addr, u32 len) -{ - const struct spi_nor_erase_map *map = &nor->params.erase_map; - const struct spi_nor_erase_type *erase, *prev_erase = NULL; - struct spi_nor_erase_region *region; - struct spi_nor_erase_command *cmd = NULL; - u64 region_end; - int ret = -EINVAL; - - region = spi_nor_find_erase_region(map, addr); - if (IS_ERR(region)) - return PTR_ERR(region); - - region_end = spi_nor_region_end(region); - - while (len) { - erase = spi_nor_find_best_erase_type(map, region, addr, len); - if (!erase) - goto destroy_erase_cmd_list; - - if (prev_erase != erase || - region->offset & SNOR_OVERLAID_REGION) { - cmd = spi_nor_init_erase_cmd(region, erase); - if (IS_ERR(cmd)) { - ret = PTR_ERR(cmd); - goto destroy_erase_cmd_list; - } - - list_add_tail(&cmd->list, erase_list); - } else { - cmd->count++; - } - - addr += cmd->size; - len -= cmd->size; - - if (len && addr >= region_end) { - region = spi_nor_region_next(region); - if (!region) - goto destroy_erase_cmd_list; - region_end = spi_nor_region_end(region); - } - - prev_erase = erase; - } - - return 0; - -destroy_erase_cmd_list: - spi_nor_destroy_erase_cmd_list(erase_list); - return ret; -} - -/** - * spi_nor_erase_multi_sectors() - perform a non-uniform erase - * @nor: pointer to a 'struct spi_nor' - * @addr: offset in the serial flash memory - * @len: number of bytes to erase - * - * Build a list of best fitted erase commands and execute it once we validate - * that the erase can be performed. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_erase_multi_sectors(struct spi_nor *nor, u64 addr, u32 len) -{ - LIST_HEAD(erase_list); - struct spi_nor_erase_command *cmd, *next; - int ret; - - ret = spi_nor_init_erase_cmd_list(nor, &erase_list, addr, len); - if (ret) - return ret; - - list_for_each_entry_safe(cmd, next, &erase_list, list) { - nor->erase_opcode = cmd->opcode; - while (cmd->count) { - ret = spi_nor_write_enable(nor); - if (ret) - goto destroy_erase_cmd_list; - - ret = spi_nor_erase_sector(nor, addr); - if (ret) - goto destroy_erase_cmd_list; - - addr += cmd->size; - cmd->count--; - - ret = spi_nor_wait_till_ready(nor); - if (ret) - goto destroy_erase_cmd_list; - } - list_del(&cmd->list); - kfree(cmd); - } - - return 0; - -destroy_erase_cmd_list: - spi_nor_destroy_erase_cmd_list(&erase_list); - return ret; -} - -/* - * Erase an address range on the nor chip. The address range may extend - * one or more erase sectors. Return an error is there is a problem erasing. - */ -static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr) -{ - struct spi_nor *nor = mtd_to_spi_nor(mtd); - u32 addr, len; - uint32_t rem; - int ret; - - dev_dbg(nor->dev, "at 0x%llx, len %lld\n", (long long)instr->addr, - (long long)instr->len); - - if (spi_nor_has_uniform_erase(nor)) { - div_u64_rem(instr->len, mtd->erasesize, &rem); - if (rem) - return -EINVAL; - } - - addr = instr->addr; - len = instr->len; - - ret = spi_nor_lock_and_prep(nor); - if (ret) - return ret; - - /* whole-chip erase? */ - if (len == mtd->size && !(nor->flags & SNOR_F_NO_OP_CHIP_ERASE)) { - unsigned long timeout; - - ret = spi_nor_write_enable(nor); - if (ret) - goto erase_err; - - ret = spi_nor_erase_chip(nor); - if (ret) - goto erase_err; - - /* - * Scale the timeout linearly with the size of the flash, with - * a minimum calibrated to an old 2MB flash. We could try to - * pull these from CFI/SFDP, but these values should be good - * enough for now. - */ - timeout = max(CHIP_ERASE_2MB_READY_WAIT_JIFFIES, - CHIP_ERASE_2MB_READY_WAIT_JIFFIES * - (unsigned long)(mtd->size / SZ_2M)); - ret = spi_nor_wait_till_ready_with_timeout(nor, timeout); - if (ret) - goto erase_err; - - /* REVISIT in some cases we could speed up erasing large regions - * by using SPINOR_OP_SE instead of SPINOR_OP_BE_4K. We may have set up - * to use "small sector erase", but that's not always optimal. - */ - - /* "sector"-at-a-time erase */ - } else if (spi_nor_has_uniform_erase(nor)) { - while (len) { - ret = spi_nor_write_enable(nor); - if (ret) - goto erase_err; - - ret = spi_nor_erase_sector(nor, addr); - if (ret) - goto erase_err; - - addr += mtd->erasesize; - len -= mtd->erasesize; - - ret = spi_nor_wait_till_ready(nor); - if (ret) - goto erase_err; - } - - /* erase multiple sectors */ - } else { - ret = spi_nor_erase_multi_sectors(nor, addr, len); - if (ret) - goto erase_err; - } - - ret = spi_nor_write_disable(nor); - -erase_err: - spi_nor_unlock_and_unprep(nor); - - return ret; -} - -static void stm_get_locked_range(struct spi_nor *nor, u8 sr, loff_t *ofs, - uint64_t *len) -{ - struct mtd_info *mtd = &nor->mtd; - u8 mask = SR_BP2 | SR_BP1 | SR_BP0; - u8 tb_mask = SR_TB_BIT5; - int shift = ffs(mask) - 1; - int pow; - - if (nor->flags & SNOR_F_HAS_SR_TB_BIT6) - tb_mask = SR_TB_BIT6; - - if (!(sr & mask)) { - /* No protection */ - *ofs = 0; - *len = 0; - } else { - pow = ((sr & mask) ^ mask) >> shift; - *len = mtd->size >> pow; - if (nor->flags & SNOR_F_HAS_SR_TB && sr & tb_mask) - *ofs = 0; - else - *ofs = mtd->size - *len; - } -} - -/* - * Return 1 if the entire region is locked (if @locked is true) or unlocked (if - * @locked is false); 0 otherwise - */ -static int stm_check_lock_status_sr(struct spi_nor *nor, loff_t ofs, uint64_t len, - u8 sr, bool locked) -{ - loff_t lock_offs; - uint64_t lock_len; - - if (!len) - return 1; - - stm_get_locked_range(nor, sr, &lock_offs, &lock_len); - - if (locked) - /* Requested range is a sub-range of locked range */ - return (ofs + len <= lock_offs + lock_len) && (ofs >= lock_offs); - else - /* Requested range does not overlap with locked range */ - return (ofs >= lock_offs + lock_len) || (ofs + len <= lock_offs); -} - -static int stm_is_locked_sr(struct spi_nor *nor, loff_t ofs, uint64_t len, - u8 sr) -{ - return stm_check_lock_status_sr(nor, ofs, len, sr, true); -} - -static int stm_is_unlocked_sr(struct spi_nor *nor, loff_t ofs, uint64_t len, - u8 sr) -{ - return stm_check_lock_status_sr(nor, ofs, len, sr, false); -} - -/* - * Lock a region of the flash. Compatible with ST Micro and similar flash. - * Supports the block protection bits BP{0,1,2} in the status register - * (SR). Does not support these features found in newer SR bitfields: - * - SEC: sector/block protect - only handle SEC=0 (block protect) - * - CMP: complement protect - only support CMP=0 (range is not complemented) - * - * Support for the following is provided conditionally for some flash: - * - TB: top/bottom protect - * - * Sample table portion for 8MB flash (Winbond w25q64fw): - * - * SEC | TB | BP2 | BP1 | BP0 | Prot Length | Protected Portion - * -------------------------------------------------------------------------- - * X | X | 0 | 0 | 0 | NONE | NONE - * 0 | 0 | 0 | 0 | 1 | 128 KB | Upper 1/64 - * 0 | 0 | 0 | 1 | 0 | 256 KB | Upper 1/32 - * 0 | 0 | 0 | 1 | 1 | 512 KB | Upper 1/16 - * 0 | 0 | 1 | 0 | 0 | 1 MB | Upper 1/8 - * 0 | 0 | 1 | 0 | 1 | 2 MB | Upper 1/4 - * 0 | 0 | 1 | 1 | 0 | 4 MB | Upper 1/2 - * X | X | 1 | 1 | 1 | 8 MB | ALL - * ------|-------|-------|-------|-------|---------------|------------------- - * 0 | 1 | 0 | 0 | 1 | 128 KB | Lower 1/64 - * 0 | 1 | 0 | 1 | 0 | 256 KB | Lower 1/32 - * 0 | 1 | 0 | 1 | 1 | 512 KB | Lower 1/16 - * 0 | 1 | 1 | 0 | 0 | 1 MB | Lower 1/8 - * 0 | 1 | 1 | 0 | 1 | 2 MB | Lower 1/4 - * 0 | 1 | 1 | 1 | 0 | 4 MB | Lower 1/2 - * - * Returns negative on errors, 0 on success. - */ -static int stm_lock(struct spi_nor *nor, loff_t ofs, uint64_t len) -{ - struct mtd_info *mtd = &nor->mtd; - int ret, status_old, status_new; - u8 mask = SR_BP2 | SR_BP1 | SR_BP0; - u8 tb_mask = SR_TB_BIT5; - u8 shift = ffs(mask) - 1, pow, val; - loff_t lock_len; - bool can_be_top = true, can_be_bottom = nor->flags & SNOR_F_HAS_SR_TB; - bool use_top; - - ret = spi_nor_read_sr(nor, nor->bouncebuf); - if (ret) - return ret; - - status_old = nor->bouncebuf[0]; - - /* If nothing in our range is unlocked, we don't need to do anything */ - if (stm_is_locked_sr(nor, ofs, len, status_old)) - return 0; - - /* If anything below us is unlocked, we can't use 'bottom' protection */ - if (!stm_is_locked_sr(nor, 0, ofs, status_old)) - can_be_bottom = false; - - /* If anything above us is unlocked, we can't use 'top' protection */ - if (!stm_is_locked_sr(nor, ofs + len, mtd->size - (ofs + len), - status_old)) - can_be_top = false; - - if (!can_be_bottom && !can_be_top) - return -EINVAL; - - /* Prefer top, if both are valid */ - use_top = can_be_top; - - /* lock_len: length of region that should end up locked */ - if (use_top) - lock_len = mtd->size - ofs; - else - lock_len = ofs + len; - - if (nor->flags & SNOR_F_HAS_SR_TB_BIT6) - tb_mask = SR_TB_BIT6; - - /* - * Need smallest pow such that: - * - * 1 / (2^pow) <= (len / size) - * - * so (assuming power-of-2 size) we do: - * - * pow = ceil(log2(size / len)) = log2(size) - floor(log2(len)) - */ - pow = ilog2(mtd->size) - ilog2(lock_len); - val = mask - (pow << shift); - if (val & ~mask) - return -EINVAL; - /* Don't "lock" with no region! */ - if (!(val & mask)) - return -EINVAL; - - status_new = (status_old & ~mask & ~tb_mask) | val; - - /* Disallow further writes if WP pin is asserted */ - status_new |= SR_SRWD; - - if (!use_top) - status_new |= tb_mask; - - /* Don't bother if they're the same */ - if (status_new == status_old) - return 0; - - /* Only modify protection if it will not unlock other areas */ - if ((status_new & mask) < (status_old & mask)) - return -EINVAL; - - return spi_nor_write_sr_and_check(nor, status_new); -} - -/* - * Unlock a region of the flash. See stm_lock() for more info - * - * Returns negative on errors, 0 on success. - */ -static int stm_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len) -{ - struct mtd_info *mtd = &nor->mtd; - int ret, status_old, status_new; - u8 mask = SR_BP2 | SR_BP1 | SR_BP0; - u8 tb_mask = SR_TB_BIT5; - u8 shift = ffs(mask) - 1, pow, val; - loff_t lock_len; - bool can_be_top = true, can_be_bottom = nor->flags & SNOR_F_HAS_SR_TB; - bool use_top; - - ret = spi_nor_read_sr(nor, nor->bouncebuf); - if (ret) - return ret; - - status_old = nor->bouncebuf[0]; - - /* If nothing in our range is locked, we don't need to do anything */ - if (stm_is_unlocked_sr(nor, ofs, len, status_old)) - return 0; - - /* If anything below us is locked, we can't use 'top' protection */ - if (!stm_is_unlocked_sr(nor, 0, ofs, status_old)) - can_be_top = false; - - /* If anything above us is locked, we can't use 'bottom' protection */ - if (!stm_is_unlocked_sr(nor, ofs + len, mtd->size - (ofs + len), - status_old)) - can_be_bottom = false; - - if (!can_be_bottom && !can_be_top) - return -EINVAL; - - /* Prefer top, if both are valid */ - use_top = can_be_top; - - /* lock_len: length of region that should remain locked */ - if (use_top) - lock_len = mtd->size - (ofs + len); - else - lock_len = ofs; - - if (nor->flags & SNOR_F_HAS_SR_TB_BIT6) - tb_mask = SR_TB_BIT6; - /* - * Need largest pow such that: - * - * 1 / (2^pow) >= (len / size) - * - * so (assuming power-of-2 size) we do: - * - * pow = floor(log2(size / len)) = log2(size) - ceil(log2(len)) - */ - pow = ilog2(mtd->size) - order_base_2(lock_len); - if (lock_len == 0) { - val = 0; /* fully unlocked */ - } else { - val = mask - (pow << shift); - /* Some power-of-two sizes are not supported */ - if (val & ~mask) - return -EINVAL; - } - - status_new = (status_old & ~mask & ~tb_mask) | val; - - /* Don't protect status register if we're fully unlocked */ - if (lock_len == 0) - status_new &= ~SR_SRWD; - - if (!use_top) - status_new |= tb_mask; - - /* Don't bother if they're the same */ - if (status_new == status_old) - return 0; - - /* Only modify protection if it will not lock other areas */ - if ((status_new & mask) > (status_old & mask)) - return -EINVAL; - - return spi_nor_write_sr_and_check(nor, status_new); -} - -/* - * Check if a region of the flash is (completely) locked. See stm_lock() for - * more info. - * - * Returns 1 if entire region is locked, 0 if any portion is unlocked, and - * negative on errors. - */ -static int stm_is_locked(struct spi_nor *nor, loff_t ofs, uint64_t len) -{ - int ret; - - ret = spi_nor_read_sr(nor, nor->bouncebuf); - if (ret) - return ret; - - return stm_is_locked_sr(nor, ofs, len, nor->bouncebuf[0]); -} - -static const struct spi_nor_locking_ops stm_locking_ops = { - .lock = stm_lock, - .unlock = stm_unlock, - .is_locked = stm_is_locked, -}; - -static int spi_nor_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) -{ - struct spi_nor *nor = mtd_to_spi_nor(mtd); - int ret; - - ret = spi_nor_lock_and_prep(nor); - if (ret) - return ret; - - ret = nor->params.locking_ops->lock(nor, ofs, len); - - spi_nor_unlock_and_unprep(nor); - return ret; -} - -static int spi_nor_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) -{ - struct spi_nor *nor = mtd_to_spi_nor(mtd); - int ret; - - ret = spi_nor_lock_and_prep(nor); - if (ret) - return ret; - - ret = nor->params.locking_ops->unlock(nor, ofs, len); - - spi_nor_unlock_and_unprep(nor); - return ret; -} - -static int spi_nor_is_locked(struct mtd_info *mtd, loff_t ofs, uint64_t len) -{ - struct spi_nor *nor = mtd_to_spi_nor(mtd); - int ret; - - ret = spi_nor_lock_and_prep(nor); - if (ret) - return ret; - - ret = nor->params.locking_ops->is_locked(nor, ofs, len); - - spi_nor_unlock_and_unprep(nor); - return ret; -} - -/** - * spi_nor_sr1_bit6_quad_enable() - Set the Quad Enable BIT(6) in the Status - * Register 1. - * @nor: pointer to a 'struct spi_nor' - * - * Bit 6 of the Status Register 1 is the QE bit for Macronix like QSPI memories. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_sr1_bit6_quad_enable(struct spi_nor *nor) -{ - int ret; - - ret = spi_nor_read_sr(nor, nor->bouncebuf); - if (ret) - return ret; - - if (nor->bouncebuf[0] & SR1_QUAD_EN_BIT6) - return 0; - - nor->bouncebuf[0] |= SR1_QUAD_EN_BIT6; - - return spi_nor_write_sr1_and_check(nor, nor->bouncebuf[0]); -} - -/** - * spi_nor_sr2_bit1_quad_enable() - set the Quad Enable BIT(1) in the Status - * Register 2. - * @nor: pointer to a 'struct spi_nor'. - * - * Bit 1 of the Status Register 2 is the QE bit for Spansion like QSPI memories. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_sr2_bit1_quad_enable(struct spi_nor *nor) -{ - int ret; - - if (nor->flags & SNOR_F_NO_READ_CR) - return spi_nor_write_16bit_cr_and_check(nor, SR2_QUAD_EN_BIT1); - - ret = spi_nor_read_cr(nor, nor->bouncebuf); - if (ret) - return ret; - - if (nor->bouncebuf[0] & SR2_QUAD_EN_BIT1) - return 0; - - nor->bouncebuf[0] |= SR2_QUAD_EN_BIT1; - - return spi_nor_write_16bit_cr_and_check(nor, nor->bouncebuf[0]); -} - -/** - * spi_nor_sr2_bit7_quad_enable() - set QE bit in Status Register 2. - * @nor: pointer to a 'struct spi_nor' - * - * Set the Quad Enable (QE) bit in the Status Register 2. - * - * This is one of the procedures to set the QE bit described in the SFDP - * (JESD216 rev B) specification but no manufacturer using this procedure has - * been identified yet, hence the name of the function. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_sr2_bit7_quad_enable(struct spi_nor *nor) -{ - u8 *sr2 = nor->bouncebuf; - int ret; - u8 sr2_written; - - /* Check current Quad Enable bit value. */ - ret = spi_nor_read_sr2(nor, sr2); - if (ret) - return ret; - if (*sr2 & SR2_QUAD_EN_BIT7) - return 0; - - /* Update the Quad Enable bit. */ - *sr2 |= SR2_QUAD_EN_BIT7; - - ret = spi_nor_write_sr2(nor, sr2); - if (ret) - return ret; - - sr2_written = *sr2; - - /* Read back and check it. */ - ret = spi_nor_read_sr2(nor, sr2); - if (ret) - return ret; - - if (*sr2 != sr2_written) { - dev_dbg(nor->dev, "SR2: Read back test failed\n"); - return -EIO; - } - - return 0; -} - -/* Used when the "_ext_id" is two bytes at most */ -#define INFO(_jedec_id, _ext_id, _sector_size, _n_sectors, _flags) \ - .id = { \ - ((_jedec_id) >> 16) & 0xff, \ - ((_jedec_id) >> 8) & 0xff, \ - (_jedec_id) & 0xff, \ - ((_ext_id) >> 8) & 0xff, \ - (_ext_id) & 0xff, \ - }, \ - .id_len = (!(_jedec_id) ? 0 : (3 + ((_ext_id) ? 2 : 0))), \ - .sector_size = (_sector_size), \ - .n_sectors = (_n_sectors), \ - .page_size = 256, \ - .flags = (_flags), - -#define INFO6(_jedec_id, _ext_id, _sector_size, _n_sectors, _flags) \ - .id = { \ - ((_jedec_id) >> 16) & 0xff, \ - ((_jedec_id) >> 8) & 0xff, \ - (_jedec_id) & 0xff, \ - ((_ext_id) >> 16) & 0xff, \ - ((_ext_id) >> 8) & 0xff, \ - (_ext_id) & 0xff, \ - }, \ - .id_len = 6, \ - .sector_size = (_sector_size), \ - .n_sectors = (_n_sectors), \ - .page_size = 256, \ - .flags = (_flags), - -#define CAT25_INFO(_sector_size, _n_sectors, _page_size, _addr_width, _flags) \ - .sector_size = (_sector_size), \ - .n_sectors = (_n_sectors), \ - .page_size = (_page_size), \ - .addr_width = (_addr_width), \ - .flags = (_flags), - -#define S3AN_INFO(_jedec_id, _n_sectors, _page_size) \ - .id = { \ - ((_jedec_id) >> 16) & 0xff, \ - ((_jedec_id) >> 8) & 0xff, \ - (_jedec_id) & 0xff \ - }, \ - .id_len = 3, \ - .sector_size = (8*_page_size), \ - .n_sectors = (_n_sectors), \ - .page_size = _page_size, \ - .addr_width = 3, \ - .flags = SPI_NOR_NO_FR | SPI_S3AN, - -static int -is25lp256_post_bfpt_fixups(struct spi_nor *nor, - const struct sfdp_parameter_header *bfpt_header, - const struct sfdp_bfpt *bfpt, - struct spi_nor_flash_parameter *params) -{ - /* - * IS25LP256 supports 4B opcodes, but the BFPT advertises a - * BFPT_DWORD1_ADDRESS_BYTES_3_ONLY address width. - * Overwrite the address width advertised by the BFPT. - */ - if ((bfpt->dwords[BFPT_DWORD(1)] & BFPT_DWORD1_ADDRESS_BYTES_MASK) == - BFPT_DWORD1_ADDRESS_BYTES_3_ONLY) - nor->addr_width = 4; - - return 0; -} - -static struct spi_nor_fixups is25lp256_fixups = { - .post_bfpt = is25lp256_post_bfpt_fixups, -}; - -static int -mx25l25635_post_bfpt_fixups(struct spi_nor *nor, - const struct sfdp_parameter_header *bfpt_header, - const struct sfdp_bfpt *bfpt, - struct spi_nor_flash_parameter *params) -{ - /* - * MX25L25635F supports 4B opcodes but MX25L25635E does not. - * Unfortunately, Macronix has re-used the same JEDEC ID for both - * variants which prevents us from defining a new entry in the parts - * table. - * We need a way to differentiate MX25L25635E and MX25L25635F, and it - * seems that the F version advertises support for Fast Read 4-4-4 in - * its BFPT table. - */ - if (bfpt->dwords[BFPT_DWORD(5)] & BFPT_DWORD5_FAST_READ_4_4_4) - nor->flags |= SNOR_F_4B_OPCODES; - - return 0; -} - -static struct spi_nor_fixups mx25l25635_fixups = { - .post_bfpt = mx25l25635_post_bfpt_fixups, -}; - -static void gd25q256_default_init(struct spi_nor *nor) -{ - /* - * Some manufacturer like GigaDevice may use different - * bit to set QE on different memories, so the MFR can't - * indicate the quad_enable method for this case, we need - * to set it in the default_init fixup hook. - */ - nor->params.quad_enable = spi_nor_sr1_bit6_quad_enable; -} - -static struct spi_nor_fixups gd25q256_fixups = { - .default_init = gd25q256_default_init, -}; - -/* NOTE: double check command sets and memory organization when you add - * more nor chips. This current list focusses on newer chips, which - * have been converging on command sets which including JEDEC ID. - * - * All newly added entries should describe *hardware* and should use SECT_4K - * (or SECT_4K_PMC) if hardware supports erasing 4 KiB sectors. For usage - * scenarios excluding small sectors there is config option that can be - * disabled: CONFIG_MTD_SPI_NOR_USE_4K_SECTORS. - * For historical (and compatibility) reasons (before we got above config) some - * old entries may be missing 4K flag. - */ -static const struct flash_info spi_nor_ids[] = { - /* Atmel -- some are (confusingly) marketed as "DataFlash" */ - { "at25fs010", INFO(0x1f6601, 0, 32 * 1024, 4, SECT_4K) }, - { "at25fs040", INFO(0x1f6604, 0, 64 * 1024, 8, SECT_4K) }, - - { "at25df041a", INFO(0x1f4401, 0, 64 * 1024, 8, SECT_4K) }, - { "at25df321", INFO(0x1f4700, 0, 64 * 1024, 64, SECT_4K) }, - { "at25df321a", INFO(0x1f4701, 0, 64 * 1024, 64, SECT_4K) }, - { "at25df641", INFO(0x1f4800, 0, 64 * 1024, 128, SECT_4K) }, - - { "at25sl321", INFO(0x1f4216, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - - { "at26f004", INFO(0x1f0400, 0, 64 * 1024, 8, SECT_4K) }, - { "at26df081a", INFO(0x1f4501, 0, 64 * 1024, 16, SECT_4K) }, - { "at26df161a", INFO(0x1f4601, 0, 64 * 1024, 32, SECT_4K) }, - { "at26df321", INFO(0x1f4700, 0, 64 * 1024, 64, SECT_4K) }, - - { "at45db081d", INFO(0x1f2500, 0, 64 * 1024, 16, SECT_4K) }, - - /* EON -- en25xxx */ - { "en25f32", INFO(0x1c3116, 0, 64 * 1024, 64, SECT_4K) }, - { "en25p32", INFO(0x1c2016, 0, 64 * 1024, 64, 0) }, - { "en25q32b", INFO(0x1c3016, 0, 64 * 1024, 64, 0) }, - { "en25p64", INFO(0x1c2017, 0, 64 * 1024, 128, 0) }, - { "en25q64", INFO(0x1c3017, 0, 64 * 1024, 128, SECT_4K) }, - { "en25q80a", INFO(0x1c3014, 0, 64 * 1024, 16, - SECT_4K | SPI_NOR_DUAL_READ) }, - { "en25qh16", INFO(0x1c7015, 0, 64 * 1024, 32, - SECT_4K | SPI_NOR_DUAL_READ) }, - { "en25qh32", INFO(0x1c7016, 0, 64 * 1024, 64, 0) }, - { "en25qh64", INFO(0x1c7017, 0, 64 * 1024, 128, - SECT_4K | SPI_NOR_DUAL_READ) }, - { "en25qh128", INFO(0x1c7018, 0, 64 * 1024, 256, 0) }, - { "en25qh256", INFO(0x1c7019, 0, 64 * 1024, 512, 0) }, - { "en25s64", INFO(0x1c3817, 0, 64 * 1024, 128, SECT_4K) }, - - /* ESMT */ - { "f25l32pa", INFO(0x8c2016, 0, 64 * 1024, 64, SECT_4K | SPI_NOR_HAS_LOCK) }, - { "f25l32qa", INFO(0x8c4116, 0, 64 * 1024, 64, SECT_4K | SPI_NOR_HAS_LOCK) }, - { "f25l64qa", INFO(0x8c4117, 0, 64 * 1024, 128, SECT_4K | SPI_NOR_HAS_LOCK) }, - - /* Everspin */ - { "mr25h128", CAT25_INFO( 16 * 1024, 1, 256, 2, SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, - { "mr25h256", CAT25_INFO( 32 * 1024, 1, 256, 2, SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, - { "mr25h10", CAT25_INFO(128 * 1024, 1, 256, 3, SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, - { "mr25h40", CAT25_INFO(512 * 1024, 1, 256, 3, SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, - - /* Fujitsu */ - { "mb85rs1mt", INFO(0x047f27, 0, 128 * 1024, 1, SPI_NOR_NO_ERASE) }, - - /* GigaDevice */ - { - "gd25q16", INFO(0xc84015, 0, 64 * 1024, 32, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "gd25q32", INFO(0xc84016, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "gd25lq32", INFO(0xc86016, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "gd25q64", INFO(0xc84017, 0, 64 * 1024, 128, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "gd25lq64c", INFO(0xc86017, 0, 64 * 1024, 128, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "gd25lq128d", INFO(0xc86018, 0, 64 * 1024, 256, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "gd25q128", INFO(0xc84018, 0, 64 * 1024, 256, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "gd25q256", INFO(0xc84019, 0, 64 * 1024, 512, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_4B_OPCODES | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | - SPI_NOR_TB_SR_BIT6) - .fixups = &gd25q256_fixups, - }, - - /* Intel/Numonyx -- xxxs33b */ - { "160s33b", INFO(0x898911, 0, 64 * 1024, 32, 0) }, - { "320s33b", INFO(0x898912, 0, 64 * 1024, 64, 0) }, - { "640s33b", INFO(0x898913, 0, 64 * 1024, 128, 0) }, - - /* ISSI */ - { "is25cd512", INFO(0x7f9d20, 0, 32 * 1024, 2, SECT_4K) }, - { "is25lq040b", INFO(0x9d4013, 0, 64 * 1024, 8, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "is25lp016d", INFO(0x9d6015, 0, 64 * 1024, 32, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "is25lp080d", INFO(0x9d6014, 0, 64 * 1024, 16, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "is25lp032", INFO(0x9d6016, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ) }, - { "is25lp064", INFO(0x9d6017, 0, 64 * 1024, 128, - SECT_4K | SPI_NOR_DUAL_READ) }, - { "is25lp128", INFO(0x9d6018, 0, 64 * 1024, 256, - SECT_4K | SPI_NOR_DUAL_READ) }, - { "is25lp256", INFO(0x9d6019, 0, 64 * 1024, 512, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_4B_OPCODES) - .fixups = &is25lp256_fixups }, - { "is25wp032", INFO(0x9d7016, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "is25wp064", INFO(0x9d7017, 0, 64 * 1024, 128, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "is25wp128", INFO(0x9d7018, 0, 64 * 1024, 256, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "is25wp256", INFO(0x9d7019, 0, 64 * 1024, 512, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_4B_OPCODES) - .fixups = &is25lp256_fixups }, - - /* Macronix */ - { "mx25l512e", INFO(0xc22010, 0, 64 * 1024, 1, SECT_4K) }, - { "mx25l2005a", INFO(0xc22012, 0, 64 * 1024, 4, SECT_4K) }, - { "mx25l4005a", INFO(0xc22013, 0, 64 * 1024, 8, SECT_4K) }, - { "mx25l8005", INFO(0xc22014, 0, 64 * 1024, 16, 0) }, - { "mx25l1606e", INFO(0xc22015, 0, 64 * 1024, 32, SECT_4K) }, - { "mx25l3205d", INFO(0xc22016, 0, 64 * 1024, 64, SECT_4K) }, - { "mx25l3255e", INFO(0xc29e16, 0, 64 * 1024, 64, SECT_4K) }, - { "mx25l6405d", INFO(0xc22017, 0, 64 * 1024, 128, SECT_4K) }, - { "mx25u2033e", INFO(0xc22532, 0, 64 * 1024, 4, SECT_4K) }, - { "mx25u3235f", INFO(0xc22536, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "mx25u4035", INFO(0xc22533, 0, 64 * 1024, 8, SECT_4K) }, - { "mx25u8035", INFO(0xc22534, 0, 64 * 1024, 16, SECT_4K) }, - { "mx25u6435f", INFO(0xc22537, 0, 64 * 1024, 128, SECT_4K) }, - { "mx25l12805d", INFO(0xc22018, 0, 64 * 1024, 256, 0) }, - { "mx25l12855e", INFO(0xc22618, 0, 64 * 1024, 256, 0) }, - { "mx25r3235f", INFO(0xc22816, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "mx25u12835f", INFO(0xc22538, 0, 64 * 1024, 256, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "mx25l25635e", INFO(0xc22019, 0, 64 * 1024, 512, - SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) - .fixups = &mx25l25635_fixups }, - { "mx25u25635f", INFO(0xc22539, 0, 64 * 1024, 512, SECT_4K | SPI_NOR_4B_OPCODES) }, - { "mx25v8035f", INFO(0xc22314, 0, 64 * 1024, 16, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "mx25l25655e", INFO(0xc22619, 0, 64 * 1024, 512, 0) }, - { "mx66l51235l", INFO(0xc2201a, 0, 64 * 1024, 1024, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, - { "mx66u51235f", INFO(0xc2253a, 0, 64 * 1024, 1024, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, - { "mx66l1g45g", INFO(0xc2201b, 0, 64 * 1024, 2048, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "mx66l1g55g", INFO(0xc2261b, 0, 64 * 1024, 2048, SPI_NOR_QUAD_READ) }, - - /* Micron <--> ST Micro */ - { "n25q016a", INFO(0x20bb15, 0, 64 * 1024, 32, SECT_4K | SPI_NOR_QUAD_READ) }, - { "n25q032", INFO(0x20ba16, 0, 64 * 1024, 64, SPI_NOR_QUAD_READ) }, - { "n25q032a", INFO(0x20bb16, 0, 64 * 1024, 64, SPI_NOR_QUAD_READ) }, - { "n25q064", INFO(0x20ba17, 0, 64 * 1024, 128, SECT_4K | SPI_NOR_QUAD_READ) }, - { "n25q064a", INFO(0x20bb17, 0, 64 * 1024, 128, SECT_4K | SPI_NOR_QUAD_READ) }, - { "n25q128a11", INFO(0x20bb18, 0, 64 * 1024, 256, SECT_4K | - USE_FSR | SPI_NOR_QUAD_READ) }, - { "n25q128a13", INFO(0x20ba18, 0, 64 * 1024, 256, SECT_4K | - USE_FSR | SPI_NOR_QUAD_READ) }, - { "mt25ql256a", INFO6(0x20ba19, 0x104400, 64 * 1024, 512, - SECT_4K | USE_FSR | SPI_NOR_DUAL_READ | - SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, - { "n25q256a", INFO(0x20ba19, 0, 64 * 1024, 512, SECT_4K | - USE_FSR | SPI_NOR_DUAL_READ | - SPI_NOR_QUAD_READ) }, - { "mt25qu256a", INFO6(0x20bb19, 0x104400, 64 * 1024, 512, - SECT_4K | USE_FSR | SPI_NOR_DUAL_READ | - SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, - { "n25q256ax1", INFO(0x20bb19, 0, 64 * 1024, 512, SECT_4K | - USE_FSR | SPI_NOR_QUAD_READ) }, - { "mt25ql512a", INFO6(0x20ba20, 0x104400, 64 * 1024, 1024, - SECT_4K | USE_FSR | SPI_NOR_DUAL_READ | - SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, - { "n25q512ax3", INFO(0x20ba20, 0, 64 * 1024, 1024, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ) }, - { "mt25qu512a", INFO6(0x20bb20, 0x104400, 64 * 1024, 1024, - SECT_4K | USE_FSR | SPI_NOR_DUAL_READ | - SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, - { "n25q512a", INFO(0x20bb20, 0, 64 * 1024, 1024, SECT_4K | - USE_FSR | SPI_NOR_QUAD_READ) }, - { "n25q00", INFO(0x20ba21, 0, 64 * 1024, 2048, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | NO_CHIP_ERASE) }, - { "n25q00a", INFO(0x20bb21, 0, 64 * 1024, 2048, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | NO_CHIP_ERASE) }, - { "mt25ql02g", INFO(0x20ba22, 0, 64 * 1024, 4096, - SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | - NO_CHIP_ERASE) }, - { "mt25qu02g", INFO(0x20bb22, 0, 64 * 1024, 4096, SECT_4K | USE_FSR | SPI_NOR_QUAD_READ | NO_CHIP_ERASE) }, - - /* Micron */ - { - "mt35xu512aba", INFO(0x2c5b1a, 0, 128 * 1024, 512, - SECT_4K | USE_FSR | SPI_NOR_OCTAL_READ | - SPI_NOR_4B_OPCODES) - }, - { "mt35xu02g", INFO(0x2c5b1c, 0, 128 * 1024, 2048, - SECT_4K | USE_FSR | SPI_NOR_OCTAL_READ | - SPI_NOR_4B_OPCODES) }, - - /* PMC */ - { "pm25lv512", INFO(0, 0, 32 * 1024, 2, SECT_4K_PMC) }, - { "pm25lv010", INFO(0, 0, 32 * 1024, 4, SECT_4K_PMC) }, - { "pm25lq032", INFO(0x7f9d46, 0, 64 * 1024, 64, SECT_4K) }, - - /* Spansion/Cypress -- single (large) sector size only, at least - * for the chips listed here (without boot sectors). - */ - { "s25sl032p", INFO(0x010215, 0x4d00, 64 * 1024, 64, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "s25sl064p", INFO(0x010216, 0x4d00, 64 * 1024, 128, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "s25fl128s0", INFO6(0x012018, 0x4d0080, 256 * 1024, 64, - SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, - { "s25fl128s1", INFO6(0x012018, 0x4d0180, 64 * 1024, 256, - SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, - { "s25fl256s0", INFO(0x010219, 0x4d00, 256 * 1024, 128, USE_CLSR) }, - { "s25fl256s1", INFO(0x010219, 0x4d01, 64 * 1024, 512, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, - { "s25fl512s", INFO6(0x010220, 0x4d0080, 256 * 1024, 256, - SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | USE_CLSR) }, - { "s25fs512s", INFO6(0x010220, 0x4d0081, 256 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, - { "s70fl01gs", INFO(0x010221, 0x4d00, 256 * 1024, 256, 0) }, - { "s25sl12800", INFO(0x012018, 0x0300, 256 * 1024, 64, 0) }, - { "s25sl12801", INFO(0x012018, 0x0301, 64 * 1024, 256, 0) }, - { "s25fl129p0", INFO(0x012018, 0x4d00, 256 * 1024, 64, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, - { "s25fl129p1", INFO(0x012018, 0x4d01, 64 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, - { "s25sl004a", INFO(0x010212, 0, 64 * 1024, 8, 0) }, - { "s25sl008a", INFO(0x010213, 0, 64 * 1024, 16, 0) }, - { "s25sl016a", INFO(0x010214, 0, 64 * 1024, 32, 0) }, - { "s25sl032a", INFO(0x010215, 0, 64 * 1024, 64, 0) }, - { "s25sl064a", INFO(0x010216, 0, 64 * 1024, 128, 0) }, - { "s25fl004k", INFO(0xef4013, 0, 64 * 1024, 8, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "s25fl008k", INFO(0xef4014, 0, 64 * 1024, 16, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "s25fl016k", INFO(0xef4015, 0, 64 * 1024, 32, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "s25fl064k", INFO(0xef4017, 0, 64 * 1024, 128, SECT_4K) }, - { "s25fl116k", INFO(0x014015, 0, 64 * 1024, 32, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "s25fl132k", INFO(0x014016, 0, 64 * 1024, 64, SECT_4K) }, - { "s25fl164k", INFO(0x014017, 0, 64 * 1024, 128, SECT_4K) }, - { "s25fl204k", INFO(0x014013, 0, 64 * 1024, 8, SECT_4K | SPI_NOR_DUAL_READ) }, - { "s25fl208k", INFO(0x014014, 0, 64 * 1024, 16, SECT_4K | SPI_NOR_DUAL_READ) }, - { "s25fl064l", INFO(0x016017, 0, 64 * 1024, 128, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, - { "s25fl128l", INFO(0x016018, 0, 64 * 1024, 256, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, - { "s25fl256l", INFO(0x016019, 0, 64 * 1024, 512, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, - - /* SST -- large erase sizes are "overlays", "sectors" are 4K */ - { "sst25vf040b", INFO(0xbf258d, 0, 64 * 1024, 8, SECT_4K | SST_WRITE) }, - { "sst25vf080b", INFO(0xbf258e, 0, 64 * 1024, 16, SECT_4K | SST_WRITE) }, - { "sst25vf016b", INFO(0xbf2541, 0, 64 * 1024, 32, SECT_4K | SST_WRITE) }, - { "sst25vf032b", INFO(0xbf254a, 0, 64 * 1024, 64, SECT_4K | SST_WRITE) }, - { "sst25vf064c", INFO(0xbf254b, 0, 64 * 1024, 128, SECT_4K) }, - { "sst25wf512", INFO(0xbf2501, 0, 64 * 1024, 1, SECT_4K | SST_WRITE) }, - { "sst25wf010", INFO(0xbf2502, 0, 64 * 1024, 2, SECT_4K | SST_WRITE) }, - { "sst25wf020", INFO(0xbf2503, 0, 64 * 1024, 4, SECT_4K | SST_WRITE) }, - { "sst25wf020a", INFO(0x621612, 0, 64 * 1024, 4, SECT_4K) }, - { "sst25wf040b", INFO(0x621613, 0, 64 * 1024, 8, SECT_4K) }, - { "sst25wf040", INFO(0xbf2504, 0, 64 * 1024, 8, SECT_4K | SST_WRITE) }, - { "sst25wf080", INFO(0xbf2505, 0, 64 * 1024, 16, SECT_4K | SST_WRITE) }, - { "sst26wf016b", INFO(0xbf2651, 0, 64 * 1024, 32, SECT_4K | - SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "sst26vf016b", INFO(0xbf2641, 0, 64 * 1024, 32, SECT_4K | - SPI_NOR_DUAL_READ) }, - { "sst26vf064b", INFO(0xbf2643, 0, 64 * 1024, 128, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - - /* ST Microelectronics -- newer production may have feature updates */ - { "m25p05", INFO(0x202010, 0, 32 * 1024, 2, 0) }, - { "m25p10", INFO(0x202011, 0, 32 * 1024, 4, 0) }, - { "m25p20", INFO(0x202012, 0, 64 * 1024, 4, 0) }, - { "m25p40", INFO(0x202013, 0, 64 * 1024, 8, 0) }, - { "m25p80", INFO(0x202014, 0, 64 * 1024, 16, 0) }, - { "m25p16", INFO(0x202015, 0, 64 * 1024, 32, 0) }, - { "m25p32", INFO(0x202016, 0, 64 * 1024, 64, 0) }, - { "m25p64", INFO(0x202017, 0, 64 * 1024, 128, 0) }, - { "m25p128", INFO(0x202018, 0, 256 * 1024, 64, 0) }, - - { "m25p05-nonjedec", INFO(0, 0, 32 * 1024, 2, 0) }, - { "m25p10-nonjedec", INFO(0, 0, 32 * 1024, 4, 0) }, - { "m25p20-nonjedec", INFO(0, 0, 64 * 1024, 4, 0) }, - { "m25p40-nonjedec", INFO(0, 0, 64 * 1024, 8, 0) }, - { "m25p80-nonjedec", INFO(0, 0, 64 * 1024, 16, 0) }, - { "m25p16-nonjedec", INFO(0, 0, 64 * 1024, 32, 0) }, - { "m25p32-nonjedec", INFO(0, 0, 64 * 1024, 64, 0) }, - { "m25p64-nonjedec", INFO(0, 0, 64 * 1024, 128, 0) }, - { "m25p128-nonjedec", INFO(0, 0, 256 * 1024, 64, 0) }, - - { "m45pe10", INFO(0x204011, 0, 64 * 1024, 2, 0) }, - { "m45pe80", INFO(0x204014, 0, 64 * 1024, 16, 0) }, - { "m45pe16", INFO(0x204015, 0, 64 * 1024, 32, 0) }, - - { "m25pe20", INFO(0x208012, 0, 64 * 1024, 4, 0) }, - { "m25pe80", INFO(0x208014, 0, 64 * 1024, 16, 0) }, - { "m25pe16", INFO(0x208015, 0, 64 * 1024, 32, SECT_4K) }, - - { "m25px16", INFO(0x207115, 0, 64 * 1024, 32, SECT_4K) }, - { "m25px32", INFO(0x207116, 0, 64 * 1024, 64, SECT_4K) }, - { "m25px32-s0", INFO(0x207316, 0, 64 * 1024, 64, SECT_4K) }, - { "m25px32-s1", INFO(0x206316, 0, 64 * 1024, 64, SECT_4K) }, - { "m25px64", INFO(0x207117, 0, 64 * 1024, 128, 0) }, - { "m25px80", INFO(0x207114, 0, 64 * 1024, 16, 0) }, - - /* Winbond -- w25x "blocks" are 64K, "sectors" are 4KiB */ - { "w25x05", INFO(0xef3010, 0, 64 * 1024, 1, SECT_4K) }, - { "w25x10", INFO(0xef3011, 0, 64 * 1024, 2, SECT_4K) }, - { "w25x20", INFO(0xef3012, 0, 64 * 1024, 4, SECT_4K) }, - { "w25x40", INFO(0xef3013, 0, 64 * 1024, 8, SECT_4K) }, - { "w25x80", INFO(0xef3014, 0, 64 * 1024, 16, SECT_4K) }, - { "w25x16", INFO(0xef3015, 0, 64 * 1024, 32, SECT_4K) }, - { - "w25q16dw", INFO(0xef6015, 0, 64 * 1024, 32, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { "w25x32", INFO(0xef3016, 0, 64 * 1024, 64, SECT_4K) }, - { - "w25q16jv-im/jm", INFO(0xef7015, 0, 64 * 1024, 32, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { "w25q20cl", INFO(0xef4012, 0, 64 * 1024, 4, SECT_4K) }, - { "w25q20bw", INFO(0xef5012, 0, 64 * 1024, 4, SECT_4K) }, - { "w25q20ew", INFO(0xef6012, 0, 64 * 1024, 4, SECT_4K) }, - { "w25q32", INFO(0xef4016, 0, 64 * 1024, 64, SECT_4K) }, - { - "w25q32dw", INFO(0xef6016, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "w25q32jv", INFO(0xef7016, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "w25q32jwm", INFO(0xef8016, 0, 64 * 1024, 64, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { "w25x64", INFO(0xef3017, 0, 64 * 1024, 128, SECT_4K) }, - { "w25q64", INFO(0xef4017, 0, 64 * 1024, 128, SECT_4K) }, - { - "w25q64dw", INFO(0xef6017, 0, 64 * 1024, 128, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "w25q128fw", INFO(0xef6018, 0, 64 * 1024, 256, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { - "w25q128jv", INFO(0xef7018, 0, 64 * 1024, 256, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) - }, - { "w25q80", INFO(0xef5014, 0, 64 * 1024, 16, SECT_4K) }, - { "w25q80bl", INFO(0xef4014, 0, 64 * 1024, 16, SECT_4K) }, - { "w25q128", INFO(0xef4018, 0, 64 * 1024, 256, SECT_4K) }, - { "w25q256", INFO(0xef4019, 0, 64 * 1024, 512, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | - SPI_NOR_4B_OPCODES) }, - { "w25q256jvm", INFO(0xef7019, 0, 64 * 1024, 512, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "w25q256jw", INFO(0xef6019, 0, 64 * 1024, 512, - SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "w25m512jv", INFO(0xef7119, 0, 64 * 1024, 1024, - SECT_4K | SPI_NOR_QUAD_READ | SPI_NOR_DUAL_READ) }, - - /* Catalyst / On Semiconductor -- non-JEDEC */ - { "cat25c11", CAT25_INFO( 16, 8, 16, 1, SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, - { "cat25c03", CAT25_INFO( 32, 8, 16, 2, SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, - { "cat25c09", CAT25_INFO( 128, 8, 32, 2, SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, - { "cat25c17", CAT25_INFO( 256, 8, 32, 2, SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, - { "cat25128", CAT25_INFO(2048, 8, 64, 2, SPI_NOR_NO_ERASE | SPI_NOR_NO_FR) }, - - /* Xilinx S3AN Internal Flash */ - { "3S50AN", S3AN_INFO(0x1f2200, 64, 264) }, - { "3S200AN", S3AN_INFO(0x1f2400, 256, 264) }, - { "3S400AN", S3AN_INFO(0x1f2400, 256, 264) }, - { "3S700AN", S3AN_INFO(0x1f2500, 512, 264) }, - { "3S1400AN", S3AN_INFO(0x1f2600, 512, 528) }, - - /* XMC (Wuhan Xinxin Semiconductor Manufacturing Corp.) */ - { "XM25QH64A", INFO(0x207017, 0, 64 * 1024, 128, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { "XM25QH128A", INFO(0x207018, 0, 64 * 1024, 256, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, - { }, -}; - -static const struct flash_info *spi_nor_read_id(struct spi_nor *nor) -{ - int tmp; - u8 *id = nor->bouncebuf; - const struct flash_info *info; - - if (nor->spimem) { - struct spi_mem_op op = - SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDID, 1), - SPI_MEM_OP_NO_ADDR, - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_IN(SPI_NOR_MAX_ID_LEN, id, 1)); - - tmp = spi_mem_exec_op(nor->spimem, &op); - } else { - tmp = nor->controller_ops->read_reg(nor, SPINOR_OP_RDID, id, - SPI_NOR_MAX_ID_LEN); - } - if (tmp) { - dev_dbg(nor->dev, "error %d reading JEDEC ID\n", tmp); - return ERR_PTR(tmp); - } - - for (tmp = 0; tmp < ARRAY_SIZE(spi_nor_ids) - 1; tmp++) { - info = &spi_nor_ids[tmp]; - if (info->id_len) { - if (!memcmp(info->id, id, info->id_len)) - return &spi_nor_ids[tmp]; - } - } - dev_err(nor->dev, "unrecognized JEDEC id bytes: %*ph\n", - SPI_NOR_MAX_ID_LEN, id); - return ERR_PTR(-ENODEV); -} - -static int spi_nor_read(struct mtd_info *mtd, loff_t from, size_t len, - size_t *retlen, u_char *buf) -{ - struct spi_nor *nor = mtd_to_spi_nor(mtd); - ssize_t ret; - - dev_dbg(nor->dev, "from 0x%08x, len %zd\n", (u32)from, len); - - ret = spi_nor_lock_and_prep(nor); - if (ret) - return ret; - - while (len) { - loff_t addr = from; - - addr = spi_nor_convert_addr(nor, addr); - - ret = spi_nor_read_data(nor, addr, len, buf); - if (ret == 0) { - /* We shouldn't see 0-length reads */ - ret = -EIO; - goto read_err; - } - if (ret < 0) - goto read_err; - - WARN_ON(ret > len); - *retlen += ret; - buf += ret; - from += ret; - len -= ret; - } - ret = 0; - -read_err: - spi_nor_unlock_and_unprep(nor); - return ret; -} - -static int sst_write(struct mtd_info *mtd, loff_t to, size_t len, - size_t *retlen, const u_char *buf) -{ - struct spi_nor *nor = mtd_to_spi_nor(mtd); - size_t actual = 0; - int ret; - - dev_dbg(nor->dev, "to 0x%08x, len %zd\n", (u32)to, len); - - ret = spi_nor_lock_and_prep(nor); - if (ret) - return ret; - - ret = spi_nor_write_enable(nor); - if (ret) - goto out; - - nor->sst_write_second = false; - - /* Start write from odd address. */ - if (to % 2) { - nor->program_opcode = SPINOR_OP_BP; - - /* write one byte. */ - ret = spi_nor_write_data(nor, to, 1, buf); - if (ret < 0) - goto out; - WARN(ret != 1, "While writing 1 byte written %i bytes\n", ret); - ret = spi_nor_wait_till_ready(nor); - if (ret) - goto out; - - to++; - actual++; - } - - /* Write out most of the data here. */ - for (; actual < len - 1; actual += 2) { - nor->program_opcode = SPINOR_OP_AAI_WP; - - /* write two bytes. */ - ret = spi_nor_write_data(nor, to, 2, buf + actual); - if (ret < 0) - goto out; - WARN(ret != 2, "While writing 2 bytes written %i bytes\n", ret); - ret = spi_nor_wait_till_ready(nor); - if (ret) - goto out; - to += 2; - nor->sst_write_second = true; - } - nor->sst_write_second = false; - - ret = spi_nor_write_disable(nor); - if (ret) - goto out; - - ret = spi_nor_wait_till_ready(nor); - if (ret) - goto out; - - /* Write out trailing byte if it exists. */ - if (actual != len) { - ret = spi_nor_write_enable(nor); - if (ret) - goto out; - - nor->program_opcode = SPINOR_OP_BP; - ret = spi_nor_write_data(nor, to, 1, buf + actual); - if (ret < 0) - goto out; - WARN(ret != 1, "While writing 1 byte written %i bytes\n", ret); - ret = spi_nor_wait_till_ready(nor); - if (ret) - goto out; - - actual += 1; - - ret = spi_nor_write_disable(nor); - } -out: - *retlen += actual; - spi_nor_unlock_and_unprep(nor); - return ret; -} - -/* - * Write an address range to the nor chip. Data must be written in - * FLASH_PAGESIZE chunks. The address range may be any size provided - * it is within the physical boundaries. - */ -static int spi_nor_write(struct mtd_info *mtd, loff_t to, size_t len, - size_t *retlen, const u_char *buf) -{ - struct spi_nor *nor = mtd_to_spi_nor(mtd); - size_t page_offset, page_remain, i; - ssize_t ret; - - dev_dbg(nor->dev, "to 0x%08x, len %zd\n", (u32)to, len); - - ret = spi_nor_lock_and_prep(nor); - if (ret) - return ret; - - for (i = 0; i < len; ) { - ssize_t written; - loff_t addr = to + i; - - /* - * If page_size is a power of two, the offset can be quickly - * calculated with an AND operation. On the other cases we - * need to do a modulus operation (more expensive). - * Power of two numbers have only one bit set and we can use - * the instruction hweight32 to detect if we need to do a - * modulus (do_div()) or not. - */ - if (hweight32(nor->page_size) == 1) { - page_offset = addr & (nor->page_size - 1); - } else { - uint64_t aux = addr; - - page_offset = do_div(aux, nor->page_size); - } - /* the size of data remaining on the first page */ - page_remain = min_t(size_t, - nor->page_size - page_offset, len - i); - - addr = spi_nor_convert_addr(nor, addr); - - ret = spi_nor_write_enable(nor); - if (ret) - goto write_err; - - ret = spi_nor_write_data(nor, addr, page_remain, buf + i); - if (ret < 0) - goto write_err; - written = ret; - - ret = spi_nor_wait_till_ready(nor); - if (ret) - goto write_err; - *retlen += written; - i += written; - } - -write_err: - spi_nor_unlock_and_unprep(nor); - return ret; -} - -static int spi_nor_check(struct spi_nor *nor) -{ - if (!nor->dev || - (!nor->spimem && !nor->controller_ops) || - (!nor->spimem && nor->controller_ops && - (!nor->controller_ops->read || - !nor->controller_ops->write || - !nor->controller_ops->read_reg || - !nor->controller_ops->write_reg))) { - pr_err("spi-nor: please fill all the necessary fields!\n"); - return -EINVAL; - } - - if (nor->spimem && nor->controller_ops) { - dev_err(nor->dev, "nor->spimem and nor->controller_ops are mutually exclusive, please set just one of them.\n"); - return -EINVAL; - } - - return 0; -} - -static int s3an_nor_setup(struct spi_nor *nor, - const struct spi_nor_hwcaps *hwcaps) -{ - int ret; - - ret = spi_nor_xread_sr(nor, nor->bouncebuf); - if (ret) - return ret; - - nor->erase_opcode = SPINOR_OP_XSE; - nor->program_opcode = SPINOR_OP_XPP; - nor->read_opcode = SPINOR_OP_READ; - nor->flags |= SNOR_F_NO_OP_CHIP_ERASE; - - /* - * This flashes have a page size of 264 or 528 bytes (known as - * Default addressing mode). It can be changed to a more standard - * Power of two mode where the page size is 256/512. This comes - * with a price: there is 3% less of space, the data is corrupted - * and the page size cannot be changed back to default addressing - * mode. - * - * The current addressing mode can be read from the XRDSR register - * and should not be changed, because is a destructive operation. - */ - if (nor->bouncebuf[0] & XSR_PAGESIZE) { - /* Flash in Power of 2 mode */ - nor->page_size = (nor->page_size == 264) ? 256 : 512; - nor->mtd.writebufsize = nor->page_size; - nor->mtd.size = 8 * nor->page_size * nor->info->n_sectors; - nor->mtd.erasesize = 8 * nor->page_size; - } else { - /* Flash in Default addressing mode */ - nor->params.convert_addr = s3an_convert_addr; - nor->mtd.erasesize = nor->info->sector_size; - } - - return 0; -} - -static void -spi_nor_set_read_settings(struct spi_nor_read_command *read, - u8 num_mode_clocks, - u8 num_wait_states, - u8 opcode, - enum spi_nor_protocol proto) -{ - read->num_mode_clocks = num_mode_clocks; - read->num_wait_states = num_wait_states; - read->opcode = opcode; - read->proto = proto; -} - -static void -spi_nor_set_pp_settings(struct spi_nor_pp_command *pp, - u8 opcode, - enum spi_nor_protocol proto) -{ - pp->opcode = opcode; - pp->proto = proto; -} - -static int spi_nor_hwcaps2cmd(u32 hwcaps, const int table[][2], size_t size) -{ - size_t i; - - for (i = 0; i < size; i++) - if (table[i][0] == (int)hwcaps) - return table[i][1]; - - return -EINVAL; -} - -static int spi_nor_hwcaps_read2cmd(u32 hwcaps) -{ - static const int hwcaps_read2cmd[][2] = { - { SNOR_HWCAPS_READ, SNOR_CMD_READ }, - { SNOR_HWCAPS_READ_FAST, SNOR_CMD_READ_FAST }, - { SNOR_HWCAPS_READ_1_1_1_DTR, SNOR_CMD_READ_1_1_1_DTR }, - { SNOR_HWCAPS_READ_1_1_2, SNOR_CMD_READ_1_1_2 }, - { SNOR_HWCAPS_READ_1_2_2, SNOR_CMD_READ_1_2_2 }, - { SNOR_HWCAPS_READ_2_2_2, SNOR_CMD_READ_2_2_2 }, - { SNOR_HWCAPS_READ_1_2_2_DTR, SNOR_CMD_READ_1_2_2_DTR }, - { SNOR_HWCAPS_READ_1_1_4, SNOR_CMD_READ_1_1_4 }, - { SNOR_HWCAPS_READ_1_4_4, SNOR_CMD_READ_1_4_4 }, - { SNOR_HWCAPS_READ_4_4_4, SNOR_CMD_READ_4_4_4 }, - { SNOR_HWCAPS_READ_1_4_4_DTR, SNOR_CMD_READ_1_4_4_DTR }, - { SNOR_HWCAPS_READ_1_1_8, SNOR_CMD_READ_1_1_8 }, - { SNOR_HWCAPS_READ_1_8_8, SNOR_CMD_READ_1_8_8 }, - { SNOR_HWCAPS_READ_8_8_8, SNOR_CMD_READ_8_8_8 }, - { SNOR_HWCAPS_READ_1_8_8_DTR, SNOR_CMD_READ_1_8_8_DTR }, - }; - - return spi_nor_hwcaps2cmd(hwcaps, hwcaps_read2cmd, - ARRAY_SIZE(hwcaps_read2cmd)); -} - -static int spi_nor_hwcaps_pp2cmd(u32 hwcaps) -{ - static const int hwcaps_pp2cmd[][2] = { - { SNOR_HWCAPS_PP, SNOR_CMD_PP }, - { SNOR_HWCAPS_PP_1_1_4, SNOR_CMD_PP_1_1_4 }, - { SNOR_HWCAPS_PP_1_4_4, SNOR_CMD_PP_1_4_4 }, - { SNOR_HWCAPS_PP_4_4_4, SNOR_CMD_PP_4_4_4 }, - { SNOR_HWCAPS_PP_1_1_8, SNOR_CMD_PP_1_1_8 }, - { SNOR_HWCAPS_PP_1_8_8, SNOR_CMD_PP_1_8_8 }, - { SNOR_HWCAPS_PP_8_8_8, SNOR_CMD_PP_8_8_8 }, - }; - - return spi_nor_hwcaps2cmd(hwcaps, hwcaps_pp2cmd, - ARRAY_SIZE(hwcaps_pp2cmd)); -} - -/* - * Serial Flash Discoverable Parameters (SFDP) parsing. - */ - -/** - * spi_nor_read_raw() - raw read of serial flash memory. read_opcode, - * addr_width and read_dummy members of the struct spi_nor - * should be previously - * set. - * @nor: pointer to a 'struct spi_nor' - * @addr: offset in the serial flash memory - * @len: number of bytes to read - * @buf: buffer where the data is copied into (dma-safe memory) - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_read_raw(struct spi_nor *nor, u32 addr, size_t len, u8 *buf) -{ - ssize_t ret; - - while (len) { - ret = spi_nor_read_data(nor, addr, len, buf); - if (ret < 0) - return ret; - if (!ret || ret > len) - return -EIO; - - buf += ret; - addr += ret; - len -= ret; - } - return 0; -} - -/** - * spi_nor_read_sfdp() - read Serial Flash Discoverable Parameters. - * @nor: pointer to a 'struct spi_nor' - * @addr: offset in the SFDP area to start reading data from - * @len: number of bytes to read - * @buf: buffer where the SFDP data are copied into (dma-safe memory) - * - * Whatever the actual numbers of bytes for address and dummy cycles are - * for (Fast) Read commands, the Read SFDP (5Ah) instruction is always - * followed by a 3-byte address and 8 dummy clock cycles. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_read_sfdp(struct spi_nor *nor, u32 addr, - size_t len, void *buf) -{ - u8 addr_width, read_opcode, read_dummy; - int ret; - - read_opcode = nor->read_opcode; - addr_width = nor->addr_width; - read_dummy = nor->read_dummy; - - nor->read_opcode = SPINOR_OP_RDSFDP; - nor->addr_width = 3; - nor->read_dummy = 8; - - ret = spi_nor_read_raw(nor, addr, len, buf); - - nor->read_opcode = read_opcode; - nor->addr_width = addr_width; - nor->read_dummy = read_dummy; - - return ret; -} - -/** - * spi_nor_spimem_check_op - check if the operation is supported - * by controller - *@nor: pointer to a 'struct spi_nor' - *@op: pointer to op template to be checked - * - * Returns 0 if operation is supported, -ENOTSUPP otherwise. - */ -static int spi_nor_spimem_check_op(struct spi_nor *nor, - struct spi_mem_op *op) -{ - /* - * First test with 4 address bytes. The opcode itself might - * be a 3B addressing opcode but we don't care, because - * SPI controller implementation should not check the opcode, - * but just the sequence. - */ - op->addr.nbytes = 4; - if (!spi_mem_supports_op(nor->spimem, op)) { - if (nor->mtd.size > SZ_16M) - return -ENOTSUPP; - - /* If flash size <= 16MB, 3 address bytes are sufficient */ - op->addr.nbytes = 3; - if (!spi_mem_supports_op(nor->spimem, op)) - return -ENOTSUPP; - } - - return 0; -} - -/** - * spi_nor_spimem_check_readop - check if the read op is supported - * by controller - *@nor: pointer to a 'struct spi_nor' - *@read: pointer to op template to be checked - * - * Returns 0 if operation is supported, -ENOTSUPP otherwise. - */ -static int spi_nor_spimem_check_readop(struct spi_nor *nor, - const struct spi_nor_read_command *read) -{ - struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(read->opcode, 1), - SPI_MEM_OP_ADDR(3, 0, 1), - SPI_MEM_OP_DUMMY(0, 1), - SPI_MEM_OP_DATA_IN(0, NULL, 1)); - - op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(read->proto); - op.addr.buswidth = spi_nor_get_protocol_addr_nbits(read->proto); - op.data.buswidth = spi_nor_get_protocol_data_nbits(read->proto); - op.dummy.buswidth = op.addr.buswidth; - op.dummy.nbytes = (read->num_mode_clocks + read->num_wait_states) * - op.dummy.buswidth / 8; - - return spi_nor_spimem_check_op(nor, &op); -} - -/** - * spi_nor_spimem_check_pp - check if the page program op is supported - * by controller - *@nor: pointer to a 'struct spi_nor' - *@pp: pointer to op template to be checked - * - * Returns 0 if operation is supported, -ENOTSUPP otherwise. - */ -static int spi_nor_spimem_check_pp(struct spi_nor *nor, - const struct spi_nor_pp_command *pp) -{ - struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(pp->opcode, 1), - SPI_MEM_OP_ADDR(3, 0, 1), - SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_OUT(0, NULL, 1)); - - op.cmd.buswidth = spi_nor_get_protocol_inst_nbits(pp->proto); - op.addr.buswidth = spi_nor_get_protocol_addr_nbits(pp->proto); - op.data.buswidth = spi_nor_get_protocol_data_nbits(pp->proto); - - return spi_nor_spimem_check_op(nor, &op); -} - -/** - * spi_nor_spimem_adjust_hwcaps - Find optimal Read/Write protocol - * based on SPI controller capabilities - * @nor: pointer to a 'struct spi_nor' - * @hwcaps: pointer to resulting capabilities after adjusting - * according to controller and flash's capability - */ -static void -spi_nor_spimem_adjust_hwcaps(struct spi_nor *nor, u32 *hwcaps) -{ - struct spi_nor_flash_parameter *params = &nor->params; - unsigned int cap; - - /* DTR modes are not supported yet, mask them all. */ - *hwcaps &= ~SNOR_HWCAPS_DTR; - - /* X-X-X modes are not supported yet, mask them all. */ - *hwcaps &= ~SNOR_HWCAPS_X_X_X; - - for (cap = 0; cap < sizeof(*hwcaps) * BITS_PER_BYTE; cap++) { - int rdidx, ppidx; - - if (!(*hwcaps & BIT(cap))) - continue; - - rdidx = spi_nor_hwcaps_read2cmd(BIT(cap)); - if (rdidx >= 0 && - spi_nor_spimem_check_readop(nor, ¶ms->reads[rdidx])) - *hwcaps &= ~BIT(cap); - - ppidx = spi_nor_hwcaps_pp2cmd(BIT(cap)); - if (ppidx < 0) - continue; - - if (spi_nor_spimem_check_pp(nor, - ¶ms->page_programs[ppidx])) - *hwcaps &= ~BIT(cap); - } -} - -/** - * spi_nor_read_sfdp_dma_unsafe() - read Serial Flash Discoverable Parameters. - * @nor: pointer to a 'struct spi_nor' - * @addr: offset in the SFDP area to start reading data from - * @len: number of bytes to read - * @buf: buffer where the SFDP data are copied into - * - * Wrap spi_nor_read_sfdp() using a kmalloc'ed bounce buffer as @buf is now not - * guaranteed to be dma-safe. - * - * Return: -ENOMEM if kmalloc() fails, the return code of spi_nor_read_sfdp() - * otherwise. - */ -static int spi_nor_read_sfdp_dma_unsafe(struct spi_nor *nor, u32 addr, - size_t len, void *buf) -{ - void *dma_safe_buf; - int ret; - - dma_safe_buf = kmalloc(len, GFP_KERNEL); - if (!dma_safe_buf) - return -ENOMEM; - - ret = spi_nor_read_sfdp(nor, addr, len, dma_safe_buf); - memcpy(buf, dma_safe_buf, len); - kfree(dma_safe_buf); - - return ret; -} - -/* Fast Read settings. */ - -static void -spi_nor_set_read_settings_from_bfpt(struct spi_nor_read_command *read, - u16 half, - enum spi_nor_protocol proto) -{ - read->num_mode_clocks = (half >> 5) & 0x07; - read->num_wait_states = (half >> 0) & 0x1f; - read->opcode = (half >> 8) & 0xff; - read->proto = proto; -} - -struct sfdp_bfpt_read { - /* The Fast Read x-y-z hardware capability in params->hwcaps.mask. */ - u32 hwcaps; - - /* - * The bit in BFPT DWORD tells us - * whether the Fast Read x-y-z command is supported. - */ - u32 supported_dword; - u32 supported_bit; - - /* - * The half-word at offset in BFPT DWORD - * encodes the op code, the number of mode clocks and the number of wait - * states to be used by Fast Read x-y-z command. - */ - u32 settings_dword; - u32 settings_shift; - - /* The SPI protocol for this Fast Read x-y-z command. */ - enum spi_nor_protocol proto; -}; - -static const struct sfdp_bfpt_read sfdp_bfpt_reads[] = { - /* Fast Read 1-1-2 */ - { - SNOR_HWCAPS_READ_1_1_2, - BFPT_DWORD(1), BIT(16), /* Supported bit */ - BFPT_DWORD(4), 0, /* Settings */ - SNOR_PROTO_1_1_2, - }, - - /* Fast Read 1-2-2 */ - { - SNOR_HWCAPS_READ_1_2_2, - BFPT_DWORD(1), BIT(20), /* Supported bit */ - BFPT_DWORD(4), 16, /* Settings */ - SNOR_PROTO_1_2_2, - }, - - /* Fast Read 2-2-2 */ - { - SNOR_HWCAPS_READ_2_2_2, - BFPT_DWORD(5), BIT(0), /* Supported bit */ - BFPT_DWORD(6), 16, /* Settings */ - SNOR_PROTO_2_2_2, - }, - - /* Fast Read 1-1-4 */ - { - SNOR_HWCAPS_READ_1_1_4, - BFPT_DWORD(1), BIT(22), /* Supported bit */ - BFPT_DWORD(3), 16, /* Settings */ - SNOR_PROTO_1_1_4, - }, - - /* Fast Read 1-4-4 */ - { - SNOR_HWCAPS_READ_1_4_4, - BFPT_DWORD(1), BIT(21), /* Supported bit */ - BFPT_DWORD(3), 0, /* Settings */ - SNOR_PROTO_1_4_4, - }, - - /* Fast Read 4-4-4 */ - { - SNOR_HWCAPS_READ_4_4_4, - BFPT_DWORD(5), BIT(4), /* Supported bit */ - BFPT_DWORD(7), 16, /* Settings */ - SNOR_PROTO_4_4_4, - }, -}; - -struct sfdp_bfpt_erase { - /* - * The half-word at offset in DWORD encodes the - * op code and erase sector size to be used by Sector Erase commands. - */ - u32 dword; - u32 shift; -}; - -static const struct sfdp_bfpt_erase sfdp_bfpt_erases[] = { - /* Erase Type 1 in DWORD8 bits[15:0] */ - {BFPT_DWORD(8), 0}, - - /* Erase Type 2 in DWORD8 bits[31:16] */ - {BFPT_DWORD(8), 16}, - - /* Erase Type 3 in DWORD9 bits[15:0] */ - {BFPT_DWORD(9), 0}, - - /* Erase Type 4 in DWORD9 bits[31:16] */ - {BFPT_DWORD(9), 16}, -}; - -/** - * spi_nor_set_erase_type() - set a SPI NOR erase type - * @erase: pointer to a structure that describes a SPI NOR erase type - * @size: the size of the sector/block erased by the erase type - * @opcode: the SPI command op code to erase the sector/block - */ -static void spi_nor_set_erase_type(struct spi_nor_erase_type *erase, - u32 size, u8 opcode) -{ - erase->size = size; - erase->opcode = opcode; - /* JEDEC JESD216B Standard imposes erase sizes to be power of 2. */ - erase->size_shift = ffs(erase->size) - 1; - erase->size_mask = (1 << erase->size_shift) - 1; -} - -/** - * spi_nor_set_erase_settings_from_bfpt() - set erase type settings from BFPT - * @erase: pointer to a structure that describes a SPI NOR erase type - * @size: the size of the sector/block erased by the erase type - * @opcode: the SPI command op code to erase the sector/block - * @i: erase type index as sorted in the Basic Flash Parameter Table - * - * The supported Erase Types will be sorted at init in ascending order, with - * the smallest Erase Type size being the first member in the erase_type array - * of the spi_nor_erase_map structure. Save the Erase Type index as sorted in - * the Basic Flash Parameter Table since it will be used later on to - * synchronize with the supported Erase Types defined in SFDP optional tables. - */ -static void -spi_nor_set_erase_settings_from_bfpt(struct spi_nor_erase_type *erase, - u32 size, u8 opcode, u8 i) -{ - erase->idx = i; - spi_nor_set_erase_type(erase, size, opcode); -} - -/** - * spi_nor_map_cmp_erase_type() - compare the map's erase types by size - * @l: member in the left half of the map's erase_type array - * @r: member in the right half of the map's erase_type array - * - * Comparison function used in the sort() call to sort in ascending order the - * map's erase types, the smallest erase type size being the first member in the - * sorted erase_type array. - * - * Return: the result of @l->size - @r->size - */ -static int spi_nor_map_cmp_erase_type(const void *l, const void *r) -{ - const struct spi_nor_erase_type *left = l, *right = r; - - return left->size - right->size; -} - -/** - * spi_nor_sort_erase_mask() - sort erase mask - * @map: the erase map of the SPI NOR - * @erase_mask: the erase type mask to be sorted - * - * Replicate the sort done for the map's erase types in BFPT: sort the erase - * mask in ascending order with the smallest erase type size starting from - * BIT(0) in the sorted erase mask. - * - * Return: sorted erase mask. - */ -static u8 spi_nor_sort_erase_mask(struct spi_nor_erase_map *map, u8 erase_mask) -{ - struct spi_nor_erase_type *erase_type = map->erase_type; - int i; - u8 sorted_erase_mask = 0; - - if (!erase_mask) - return 0; - - /* Replicate the sort done for the map's erase types. */ - for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) - if (erase_type[i].size && erase_mask & BIT(erase_type[i].idx)) - sorted_erase_mask |= BIT(i); - - return sorted_erase_mask; -} - -/** - * spi_nor_regions_sort_erase_types() - sort erase types in each region - * @map: the erase map of the SPI NOR - * - * Function assumes that the erase types defined in the erase map are already - * sorted in ascending order, with the smallest erase type size being the first - * member in the erase_type array. It replicates the sort done for the map's - * erase types. Each region's erase bitmask will indicate which erase types are - * supported from the sorted erase types defined in the erase map. - * Sort the all region's erase type at init in order to speed up the process of - * finding the best erase command at runtime. - */ -static void spi_nor_regions_sort_erase_types(struct spi_nor_erase_map *map) -{ - struct spi_nor_erase_region *region = map->regions; - u8 region_erase_mask, sorted_erase_mask; - - while (region) { - region_erase_mask = region->offset & SNOR_ERASE_TYPE_MASK; - - sorted_erase_mask = spi_nor_sort_erase_mask(map, - region_erase_mask); - - /* Overwrite erase mask. */ - region->offset = (region->offset & ~SNOR_ERASE_TYPE_MASK) | - sorted_erase_mask; - - region = spi_nor_region_next(region); - } -} - -/** - * spi_nor_init_uniform_erase_map() - Initialize uniform erase map - * @map: the erase map of the SPI NOR - * @erase_mask: bitmask encoding erase types that can erase the entire - * flash memory - * @flash_size: the spi nor flash memory size - */ -static void spi_nor_init_uniform_erase_map(struct spi_nor_erase_map *map, - u8 erase_mask, u64 flash_size) -{ - /* Offset 0 with erase_mask and SNOR_LAST_REGION bit set */ - map->uniform_region.offset = (erase_mask & SNOR_ERASE_TYPE_MASK) | - SNOR_LAST_REGION; - map->uniform_region.size = flash_size; - map->regions = &map->uniform_region; - map->uniform_erase_type = erase_mask; -} - -static int -spi_nor_post_bfpt_fixups(struct spi_nor *nor, - const struct sfdp_parameter_header *bfpt_header, - const struct sfdp_bfpt *bfpt, - struct spi_nor_flash_parameter *params) -{ - if (nor->info->fixups && nor->info->fixups->post_bfpt) - return nor->info->fixups->post_bfpt(nor, bfpt_header, bfpt, - params); - - return 0; -} - -/** - * spi_nor_parse_bfpt() - read and parse the Basic Flash Parameter Table. - * @nor: pointer to a 'struct spi_nor' - * @bfpt_header: pointer to the 'struct sfdp_parameter_header' describing - * the Basic Flash Parameter Table length and version - * @params: pointer to the 'struct spi_nor_flash_parameter' to be - * filled - * - * The Basic Flash Parameter Table is the main and only mandatory table as - * defined by the SFDP (JESD216) specification. - * It provides us with the total size (memory density) of the data array and - * the number of address bytes for Fast Read, Page Program and Sector Erase - * commands. - * For Fast READ commands, it also gives the number of mode clock cycles and - * wait states (regrouped in the number of dummy clock cycles) for each - * supported instruction op code. - * For Page Program, the page size is now available since JESD216 rev A, however - * the supported instruction op codes are still not provided. - * For Sector Erase commands, this table stores the supported instruction op - * codes and the associated sector sizes. - * Finally, the Quad Enable Requirements (QER) are also available since JESD216 - * rev A. The QER bits encode the manufacturer dependent procedure to be - * executed to set the Quad Enable (QE) bit in some internal register of the - * Quad SPI memory. Indeed the QE bit, when it exists, must be set before - * sending any Quad SPI command to the memory. Actually, setting the QE bit - * tells the memory to reassign its WP# and HOLD#/RESET# pins to functions IO2 - * and IO3 hence enabling 4 (Quad) I/O lines. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_parse_bfpt(struct spi_nor *nor, - const struct sfdp_parameter_header *bfpt_header, - struct spi_nor_flash_parameter *params) -{ - struct spi_nor_erase_map *map = ¶ms->erase_map; - struct spi_nor_erase_type *erase_type = map->erase_type; - struct sfdp_bfpt bfpt; - size_t len; - int i, cmd, err; - u32 addr; - u16 half; - u8 erase_mask; - - /* JESD216 Basic Flash Parameter Table length is at least 9 DWORDs. */ - if (bfpt_header->length < BFPT_DWORD_MAX_JESD216) - return -EINVAL; - - /* Read the Basic Flash Parameter Table. */ - len = min_t(size_t, sizeof(bfpt), - bfpt_header->length * sizeof(u32)); - addr = SFDP_PARAM_HEADER_PTP(bfpt_header); - memset(&bfpt, 0, sizeof(bfpt)); - err = spi_nor_read_sfdp_dma_unsafe(nor, addr, len, &bfpt); - if (err < 0) - return err; - - /* Fix endianness of the BFPT DWORDs. */ - for (i = 0; i < BFPT_DWORD_MAX; i++) - bfpt.dwords[i] = le32_to_cpu(bfpt.dwords[i]); - - /* Number of address bytes. */ - switch (bfpt.dwords[BFPT_DWORD(1)] & BFPT_DWORD1_ADDRESS_BYTES_MASK) { - case BFPT_DWORD1_ADDRESS_BYTES_3_ONLY: - nor->addr_width = 3; - break; - - case BFPT_DWORD1_ADDRESS_BYTES_4_ONLY: - nor->addr_width = 4; - break; - - default: - break; - } - - /* Flash Memory Density (in bits). */ - params->size = bfpt.dwords[BFPT_DWORD(2)]; - if (params->size & BIT(31)) { - params->size &= ~BIT(31); - - /* - * Prevent overflows on params->size. Anyway, a NOR of 2^64 - * bits is unlikely to exist so this error probably means - * the BFPT we are reading is corrupted/wrong. - */ - if (params->size > 63) - return -EINVAL; - - params->size = 1ULL << params->size; - } else { - params->size++; - } - params->size >>= 3; /* Convert to bytes. */ - - /* Fast Read settings. */ - for (i = 0; i < ARRAY_SIZE(sfdp_bfpt_reads); i++) { - const struct sfdp_bfpt_read *rd = &sfdp_bfpt_reads[i]; - struct spi_nor_read_command *read; - - if (!(bfpt.dwords[rd->supported_dword] & rd->supported_bit)) { - params->hwcaps.mask &= ~rd->hwcaps; - continue; - } - - params->hwcaps.mask |= rd->hwcaps; - cmd = spi_nor_hwcaps_read2cmd(rd->hwcaps); - read = ¶ms->reads[cmd]; - half = bfpt.dwords[rd->settings_dword] >> rd->settings_shift; - spi_nor_set_read_settings_from_bfpt(read, half, rd->proto); - } - - /* - * Sector Erase settings. Reinitialize the uniform erase map using the - * Erase Types defined in the bfpt table. - */ - erase_mask = 0; - memset(¶ms->erase_map, 0, sizeof(params->erase_map)); - for (i = 0; i < ARRAY_SIZE(sfdp_bfpt_erases); i++) { - const struct sfdp_bfpt_erase *er = &sfdp_bfpt_erases[i]; - u32 erasesize; - u8 opcode; - - half = bfpt.dwords[er->dword] >> er->shift; - erasesize = half & 0xff; - - /* erasesize == 0 means this Erase Type is not supported. */ - if (!erasesize) - continue; - - erasesize = 1U << erasesize; - opcode = (half >> 8) & 0xff; - erase_mask |= BIT(i); - spi_nor_set_erase_settings_from_bfpt(&erase_type[i], erasesize, - opcode, i); - } - spi_nor_init_uniform_erase_map(map, erase_mask, params->size); - /* - * Sort all the map's Erase Types in ascending order with the smallest - * erase size being the first member in the erase_type array. - */ - sort(erase_type, SNOR_ERASE_TYPE_MAX, sizeof(erase_type[0]), - spi_nor_map_cmp_erase_type, NULL); - /* - * Sort the erase types in the uniform region in order to update the - * uniform_erase_type bitmask. The bitmask will be used later on when - * selecting the uniform erase. - */ - spi_nor_regions_sort_erase_types(map); - map->uniform_erase_type = map->uniform_region.offset & - SNOR_ERASE_TYPE_MASK; - - /* Stop here if not JESD216 rev A or later. */ - if (bfpt_header->length < BFPT_DWORD_MAX) - return spi_nor_post_bfpt_fixups(nor, bfpt_header, &bfpt, - params); - - /* Page size: this field specifies 'N' so the page size = 2^N bytes. */ - params->page_size = bfpt.dwords[BFPT_DWORD(11)]; - params->page_size &= BFPT_DWORD11_PAGE_SIZE_MASK; - params->page_size >>= BFPT_DWORD11_PAGE_SIZE_SHIFT; - params->page_size = 1U << params->page_size; - - /* Quad Enable Requirements. */ - switch (bfpt.dwords[BFPT_DWORD(15)] & BFPT_DWORD15_QER_MASK) { - case BFPT_DWORD15_QER_NONE: - params->quad_enable = NULL; - break; - - case BFPT_DWORD15_QER_SR2_BIT1_BUGGY: - /* - * Writing only one byte to the Status Register has the - * side-effect of clearing Status Register 2. - */ - case BFPT_DWORD15_QER_SR2_BIT1_NO_RD: - /* - * Read Configuration Register (35h) instruction is not - * supported. - */ - nor->flags |= SNOR_F_HAS_16BIT_SR | SNOR_F_NO_READ_CR; - params->quad_enable = spi_nor_sr2_bit1_quad_enable; - break; - - case BFPT_DWORD15_QER_SR1_BIT6: - nor->flags &= ~SNOR_F_HAS_16BIT_SR; - params->quad_enable = spi_nor_sr1_bit6_quad_enable; - break; - - case BFPT_DWORD15_QER_SR2_BIT7: - nor->flags &= ~SNOR_F_HAS_16BIT_SR; - params->quad_enable = spi_nor_sr2_bit7_quad_enable; - break; - - case BFPT_DWORD15_QER_SR2_BIT1: - /* - * JESD216 rev B or later does not specify if writing only one - * byte to the Status Register clears or not the Status - * Register 2, so let's be cautious and keep the default - * assumption of a 16-bit Write Status (01h) command. - */ - nor->flags |= SNOR_F_HAS_16BIT_SR; - - params->quad_enable = spi_nor_sr2_bit1_quad_enable; - break; - - default: - return -EINVAL; - } - - return spi_nor_post_bfpt_fixups(nor, bfpt_header, &bfpt, params); -} - -#define SMPT_CMD_ADDRESS_LEN_MASK GENMASK(23, 22) -#define SMPT_CMD_ADDRESS_LEN_0 (0x0UL << 22) -#define SMPT_CMD_ADDRESS_LEN_3 (0x1UL << 22) -#define SMPT_CMD_ADDRESS_LEN_4 (0x2UL << 22) -#define SMPT_CMD_ADDRESS_LEN_USE_CURRENT (0x3UL << 22) - -#define SMPT_CMD_READ_DUMMY_MASK GENMASK(19, 16) -#define SMPT_CMD_READ_DUMMY_SHIFT 16 -#define SMPT_CMD_READ_DUMMY(_cmd) \ - (((_cmd) & SMPT_CMD_READ_DUMMY_MASK) >> SMPT_CMD_READ_DUMMY_SHIFT) -#define SMPT_CMD_READ_DUMMY_IS_VARIABLE 0xfUL - -#define SMPT_CMD_READ_DATA_MASK GENMASK(31, 24) -#define SMPT_CMD_READ_DATA_SHIFT 24 -#define SMPT_CMD_READ_DATA(_cmd) \ - (((_cmd) & SMPT_CMD_READ_DATA_MASK) >> SMPT_CMD_READ_DATA_SHIFT) - -#define SMPT_CMD_OPCODE_MASK GENMASK(15, 8) -#define SMPT_CMD_OPCODE_SHIFT 8 -#define SMPT_CMD_OPCODE(_cmd) \ - (((_cmd) & SMPT_CMD_OPCODE_MASK) >> SMPT_CMD_OPCODE_SHIFT) - -#define SMPT_MAP_REGION_COUNT_MASK GENMASK(23, 16) -#define SMPT_MAP_REGION_COUNT_SHIFT 16 -#define SMPT_MAP_REGION_COUNT(_header) \ - ((((_header) & SMPT_MAP_REGION_COUNT_MASK) >> \ - SMPT_MAP_REGION_COUNT_SHIFT) + 1) - -#define SMPT_MAP_ID_MASK GENMASK(15, 8) -#define SMPT_MAP_ID_SHIFT 8 -#define SMPT_MAP_ID(_header) \ - (((_header) & SMPT_MAP_ID_MASK) >> SMPT_MAP_ID_SHIFT) - -#define SMPT_MAP_REGION_SIZE_MASK GENMASK(31, 8) -#define SMPT_MAP_REGION_SIZE_SHIFT 8 -#define SMPT_MAP_REGION_SIZE(_region) \ - (((((_region) & SMPT_MAP_REGION_SIZE_MASK) >> \ - SMPT_MAP_REGION_SIZE_SHIFT) + 1) * 256) - -#define SMPT_MAP_REGION_ERASE_TYPE_MASK GENMASK(3, 0) -#define SMPT_MAP_REGION_ERASE_TYPE(_region) \ - ((_region) & SMPT_MAP_REGION_ERASE_TYPE_MASK) - -#define SMPT_DESC_TYPE_MAP BIT(1) -#define SMPT_DESC_END BIT(0) - -/** - * spi_nor_smpt_addr_width() - return the address width used in the - * configuration detection command. - * @nor: pointer to a 'struct spi_nor' - * @settings: configuration detection command descriptor, dword1 - */ -static u8 spi_nor_smpt_addr_width(const struct spi_nor *nor, const u32 settings) -{ - switch (settings & SMPT_CMD_ADDRESS_LEN_MASK) { - case SMPT_CMD_ADDRESS_LEN_0: - return 0; - case SMPT_CMD_ADDRESS_LEN_3: - return 3; - case SMPT_CMD_ADDRESS_LEN_4: - return 4; - case SMPT_CMD_ADDRESS_LEN_USE_CURRENT: - /* fall through */ - default: - return nor->addr_width; - } -} - -/** - * spi_nor_smpt_read_dummy() - return the configuration detection command read - * latency, in clock cycles. - * @nor: pointer to a 'struct spi_nor' - * @settings: configuration detection command descriptor, dword1 - * - * Return: the number of dummy cycles for an SMPT read - */ -static u8 spi_nor_smpt_read_dummy(const struct spi_nor *nor, const u32 settings) -{ - u8 read_dummy = SMPT_CMD_READ_DUMMY(settings); - - if (read_dummy == SMPT_CMD_READ_DUMMY_IS_VARIABLE) - return nor->read_dummy; - return read_dummy; -} - -/** - * spi_nor_get_map_in_use() - get the configuration map in use - * @nor: pointer to a 'struct spi_nor' - * @smpt: pointer to the sector map parameter table - * @smpt_len: sector map parameter table length - * - * Return: pointer to the map in use, ERR_PTR(-errno) otherwise. - */ -static const u32 *spi_nor_get_map_in_use(struct spi_nor *nor, const u32 *smpt, - u8 smpt_len) -{ - const u32 *ret; - u8 *buf; - u32 addr; - int err; - u8 i; - u8 addr_width, read_opcode, read_dummy; - u8 read_data_mask, map_id; - - /* Use a kmalloc'ed bounce buffer to guarantee it is DMA-able. */ - buf = kmalloc(sizeof(*buf), GFP_KERNEL); - if (!buf) - return ERR_PTR(-ENOMEM); - - addr_width = nor->addr_width; - read_dummy = nor->read_dummy; - read_opcode = nor->read_opcode; - - map_id = 0; - /* Determine if there are any optional Detection Command Descriptors */ - for (i = 0; i < smpt_len; i += 2) { - if (smpt[i] & SMPT_DESC_TYPE_MAP) - break; - - read_data_mask = SMPT_CMD_READ_DATA(smpt[i]); - nor->addr_width = spi_nor_smpt_addr_width(nor, smpt[i]); - nor->read_dummy = spi_nor_smpt_read_dummy(nor, smpt[i]); - nor->read_opcode = SMPT_CMD_OPCODE(smpt[i]); - addr = smpt[i + 1]; - - err = spi_nor_read_raw(nor, addr, 1, buf); - if (err) { - ret = ERR_PTR(err); - goto out; - } - - /* - * Build an index value that is used to select the Sector Map - * Configuration that is currently in use. - */ - map_id = map_id << 1 | !!(*buf & read_data_mask); - } - - /* - * If command descriptors are provided, they always precede map - * descriptors in the table. There is no need to start the iteration - * over smpt array all over again. - * - * Find the matching configuration map. - */ - ret = ERR_PTR(-EINVAL); - while (i < smpt_len) { - if (SMPT_MAP_ID(smpt[i]) == map_id) { - ret = smpt + i; - break; - } - - /* - * If there are no more configuration map descriptors and no - * configuration ID matched the configuration identifier, the - * sector address map is unknown. - */ - if (smpt[i] & SMPT_DESC_END) - break; - - /* increment the table index to the next map */ - i += SMPT_MAP_REGION_COUNT(smpt[i]) + 1; - } - - /* fall through */ -out: - kfree(buf); - nor->addr_width = addr_width; - nor->read_dummy = read_dummy; - nor->read_opcode = read_opcode; - return ret; -} - -/** - * spi_nor_region_check_overlay() - set overlay bit when the region is overlaid - * @region: pointer to a structure that describes a SPI NOR erase region - * @erase: pointer to a structure that describes a SPI NOR erase type - * @erase_type: erase type bitmask - */ -static void -spi_nor_region_check_overlay(struct spi_nor_erase_region *region, - const struct spi_nor_erase_type *erase, - const u8 erase_type) -{ - int i; - - for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { - if (!(erase_type & BIT(i))) - continue; - if (region->size & erase[i].size_mask) { - spi_nor_region_mark_overlay(region); - return; - } - } -} - -/** - * spi_nor_init_non_uniform_erase_map() - initialize the non-uniform erase map - * @nor: pointer to a 'struct spi_nor' - * @params: pointer to a duplicate 'struct spi_nor_flash_parameter' that is - * used for storing SFDP parsed data - * @smpt: pointer to the sector map parameter table - * - * Return: 0 on success, -errno otherwise. - */ -static int -spi_nor_init_non_uniform_erase_map(struct spi_nor *nor, - struct spi_nor_flash_parameter *params, - const u32 *smpt) -{ - struct spi_nor_erase_map *map = ¶ms->erase_map; - struct spi_nor_erase_type *erase = map->erase_type; - struct spi_nor_erase_region *region; - u64 offset; - u32 region_count; - int i, j; - u8 uniform_erase_type, save_uniform_erase_type; - u8 erase_type, regions_erase_type; - - region_count = SMPT_MAP_REGION_COUNT(*smpt); - /* - * The regions will be freed when the driver detaches from the - * device. - */ - region = devm_kcalloc(nor->dev, region_count, sizeof(*region), - GFP_KERNEL); - if (!region) - return -ENOMEM; - map->regions = region; - - uniform_erase_type = 0xff; - regions_erase_type = 0; - offset = 0; - /* Populate regions. */ - for (i = 0; i < region_count; i++) { - j = i + 1; /* index for the region dword */ - region[i].size = SMPT_MAP_REGION_SIZE(smpt[j]); - erase_type = SMPT_MAP_REGION_ERASE_TYPE(smpt[j]); - region[i].offset = offset | erase_type; - - spi_nor_region_check_overlay(®ion[i], erase, erase_type); - - /* - * Save the erase types that are supported in all regions and - * can erase the entire flash memory. - */ - uniform_erase_type &= erase_type; - - /* - * regions_erase_type mask will indicate all the erase types - * supported in this configuration map. - */ - regions_erase_type |= erase_type; - - offset = (region[i].offset & ~SNOR_ERASE_FLAGS_MASK) + - region[i].size; - } - - save_uniform_erase_type = map->uniform_erase_type; - map->uniform_erase_type = spi_nor_sort_erase_mask(map, - uniform_erase_type); - - if (!regions_erase_type) { - /* - * Roll back to the previous uniform_erase_type mask, SMPT is - * broken. - */ - map->uniform_erase_type = save_uniform_erase_type; - return -EINVAL; - } - - /* - * BFPT advertises all the erase types supported by all the possible - * map configurations. Mask out the erase types that are not supported - * by the current map configuration. - */ - for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) - if (!(regions_erase_type & BIT(erase[i].idx))) - spi_nor_set_erase_type(&erase[i], 0, 0xFF); - - spi_nor_region_mark_end(®ion[i - 1]); - - return 0; -} - -/** - * spi_nor_parse_smpt() - parse Sector Map Parameter Table - * @nor: pointer to a 'struct spi_nor' - * @smpt_header: sector map parameter table header - * @params: pointer to a duplicate 'struct spi_nor_flash_parameter' - * that is used for storing SFDP parsed data - * - * This table is optional, but when available, we parse it to identify the - * location and size of sectors within the main data array of the flash memory - * device and to identify which Erase Types are supported by each sector. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_parse_smpt(struct spi_nor *nor, - const struct sfdp_parameter_header *smpt_header, - struct spi_nor_flash_parameter *params) -{ - const u32 *sector_map; - u32 *smpt; - size_t len; - u32 addr; - int i, ret; - - /* Read the Sector Map Parameter Table. */ - len = smpt_header->length * sizeof(*smpt); - smpt = kmalloc(len, GFP_KERNEL); - if (!smpt) - return -ENOMEM; - - addr = SFDP_PARAM_HEADER_PTP(smpt_header); - ret = spi_nor_read_sfdp(nor, addr, len, smpt); - if (ret) - goto out; - - /* Fix endianness of the SMPT DWORDs. */ - for (i = 0; i < smpt_header->length; i++) - smpt[i] = le32_to_cpu(smpt[i]); - - sector_map = spi_nor_get_map_in_use(nor, smpt, smpt_header->length); - if (IS_ERR(sector_map)) { - ret = PTR_ERR(sector_map); - goto out; - } - - ret = spi_nor_init_non_uniform_erase_map(nor, params, sector_map); - if (ret) - goto out; - - spi_nor_regions_sort_erase_types(¶ms->erase_map); - /* fall through */ -out: - kfree(smpt); - return ret; -} - -#define SFDP_4BAIT_DWORD_MAX 2 - -struct sfdp_4bait { - /* The hardware capability. */ - u32 hwcaps; - - /* - * The bit in DWORD1 of the 4BAIT tells us whether - * the associated 4-byte address op code is supported. - */ - u32 supported_bit; -}; - -/** - * spi_nor_parse_4bait() - parse the 4-Byte Address Instruction Table - * @nor: pointer to a 'struct spi_nor'. - * @param_header: pointer to the 'struct sfdp_parameter_header' describing - * the 4-Byte Address Instruction Table length and version. - * @params: pointer to the 'struct spi_nor_flash_parameter' to be. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_parse_4bait(struct spi_nor *nor, - const struct sfdp_parameter_header *param_header, - struct spi_nor_flash_parameter *params) -{ - static const struct sfdp_4bait reads[] = { - { SNOR_HWCAPS_READ, BIT(0) }, - { SNOR_HWCAPS_READ_FAST, BIT(1) }, - { SNOR_HWCAPS_READ_1_1_2, BIT(2) }, - { SNOR_HWCAPS_READ_1_2_2, BIT(3) }, - { SNOR_HWCAPS_READ_1_1_4, BIT(4) }, - { SNOR_HWCAPS_READ_1_4_4, BIT(5) }, - { SNOR_HWCAPS_READ_1_1_1_DTR, BIT(13) }, - { SNOR_HWCAPS_READ_1_2_2_DTR, BIT(14) }, - { SNOR_HWCAPS_READ_1_4_4_DTR, BIT(15) }, - }; - static const struct sfdp_4bait programs[] = { - { SNOR_HWCAPS_PP, BIT(6) }, - { SNOR_HWCAPS_PP_1_1_4, BIT(7) }, - { SNOR_HWCAPS_PP_1_4_4, BIT(8) }, - }; - static const struct sfdp_4bait erases[SNOR_ERASE_TYPE_MAX] = { - { 0u /* not used */, BIT(9) }, - { 0u /* not used */, BIT(10) }, - { 0u /* not used */, BIT(11) }, - { 0u /* not used */, BIT(12) }, - }; - struct spi_nor_pp_command *params_pp = params->page_programs; - struct spi_nor_erase_map *map = ¶ms->erase_map; - struct spi_nor_erase_type *erase_type = map->erase_type; - u32 *dwords; - size_t len; - u32 addr, discard_hwcaps, read_hwcaps, pp_hwcaps, erase_mask; - int i, ret; - - if (param_header->major != SFDP_JESD216_MAJOR || - param_header->length < SFDP_4BAIT_DWORD_MAX) - return -EINVAL; - - /* Read the 4-byte Address Instruction Table. */ - len = sizeof(*dwords) * SFDP_4BAIT_DWORD_MAX; - - /* Use a kmalloc'ed bounce buffer to guarantee it is DMA-able. */ - dwords = kmalloc(len, GFP_KERNEL); - if (!dwords) - return -ENOMEM; - - addr = SFDP_PARAM_HEADER_PTP(param_header); - ret = spi_nor_read_sfdp(nor, addr, len, dwords); - if (ret) - goto out; - - /* Fix endianness of the 4BAIT DWORDs. */ - for (i = 0; i < SFDP_4BAIT_DWORD_MAX; i++) - dwords[i] = le32_to_cpu(dwords[i]); - - /* - * Compute the subset of (Fast) Read commands for which the 4-byte - * version is supported. - */ - discard_hwcaps = 0; - read_hwcaps = 0; - for (i = 0; i < ARRAY_SIZE(reads); i++) { - const struct sfdp_4bait *read = &reads[i]; - - discard_hwcaps |= read->hwcaps; - if ((params->hwcaps.mask & read->hwcaps) && - (dwords[0] & read->supported_bit)) - read_hwcaps |= read->hwcaps; - } - - /* - * Compute the subset of Page Program commands for which the 4-byte - * version is supported. - */ - pp_hwcaps = 0; - for (i = 0; i < ARRAY_SIZE(programs); i++) { - const struct sfdp_4bait *program = &programs[i]; - - /* - * The 4 Byte Address Instruction (Optional) Table is the only - * SFDP table that indicates support for Page Program Commands. - * Bypass the params->hwcaps.mask and consider 4BAIT the biggest - * authority for specifying Page Program support. - */ - discard_hwcaps |= program->hwcaps; - if (dwords[0] & program->supported_bit) - pp_hwcaps |= program->hwcaps; - } - - /* - * Compute the subset of Sector Erase commands for which the 4-byte - * version is supported. - */ - erase_mask = 0; - for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { - const struct sfdp_4bait *erase = &erases[i]; - - if (dwords[0] & erase->supported_bit) - erase_mask |= BIT(i); - } - - /* Replicate the sort done for the map's erase types in BFPT. */ - erase_mask = spi_nor_sort_erase_mask(map, erase_mask); - - /* - * We need at least one 4-byte op code per read, program and erase - * operation; the .read(), .write() and .erase() hooks share the - * nor->addr_width value. - */ - if (!read_hwcaps || !pp_hwcaps || !erase_mask) - goto out; - - /* - * Discard all operations from the 4-byte instruction set which are - * not supported by this memory. - */ - params->hwcaps.mask &= ~discard_hwcaps; - params->hwcaps.mask |= (read_hwcaps | pp_hwcaps); - - /* Use the 4-byte address instruction set. */ - for (i = 0; i < SNOR_CMD_READ_MAX; i++) { - struct spi_nor_read_command *read_cmd = ¶ms->reads[i]; - - read_cmd->opcode = spi_nor_convert_3to4_read(read_cmd->opcode); - } - - /* 4BAIT is the only SFDP table that indicates page program support. */ - if (pp_hwcaps & SNOR_HWCAPS_PP) - spi_nor_set_pp_settings(¶ms_pp[SNOR_CMD_PP], - SPINOR_OP_PP_4B, SNOR_PROTO_1_1_1); - if (pp_hwcaps & SNOR_HWCAPS_PP_1_1_4) - spi_nor_set_pp_settings(¶ms_pp[SNOR_CMD_PP_1_1_4], - SPINOR_OP_PP_1_1_4_4B, - SNOR_PROTO_1_1_4); - if (pp_hwcaps & SNOR_HWCAPS_PP_1_4_4) - spi_nor_set_pp_settings(¶ms_pp[SNOR_CMD_PP_1_4_4], - SPINOR_OP_PP_1_4_4_4B, - SNOR_PROTO_1_4_4); - - for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) { - if (erase_mask & BIT(i)) - erase_type[i].opcode = (dwords[1] >> - erase_type[i].idx * 8) & 0xFF; - else - spi_nor_set_erase_type(&erase_type[i], 0u, 0xFF); - } - - /* - * We set SNOR_F_HAS_4BAIT in order to skip spi_nor_set_4byte_opcodes() - * later because we already did the conversion to 4byte opcodes. Also, - * this latest function implements a legacy quirk for the erase size of - * Spansion memory. However this quirk is no longer needed with new - * SFDP compliant memories. - */ - nor->addr_width = 4; - nor->flags |= SNOR_F_4B_OPCODES | SNOR_F_HAS_4BAIT; - - /* fall through */ -out: - kfree(dwords); - return ret; -} - -/** - * spi_nor_parse_sfdp() - parse the Serial Flash Discoverable Parameters. - * @nor: pointer to a 'struct spi_nor' - * @params: pointer to the 'struct spi_nor_flash_parameter' to be - * filled - * - * The Serial Flash Discoverable Parameters are described by the JEDEC JESD216 - * specification. This is a standard which tends to supported by almost all - * (Q)SPI memory manufacturers. Those hard-coded tables allow us to learn at - * runtime the main parameters needed to perform basic SPI flash operations such - * as Fast Read, Page Program or Sector Erase commands. - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_parse_sfdp(struct spi_nor *nor, - struct spi_nor_flash_parameter *params) -{ - const struct sfdp_parameter_header *param_header, *bfpt_header; - struct sfdp_parameter_header *param_headers = NULL; - struct sfdp_header header; - struct device *dev = nor->dev; - size_t psize; - int i, err; - - /* Get the SFDP header. */ - err = spi_nor_read_sfdp_dma_unsafe(nor, 0, sizeof(header), &header); - if (err < 0) - return err; - - /* Check the SFDP header version. */ - if (le32_to_cpu(header.signature) != SFDP_SIGNATURE || - header.major != SFDP_JESD216_MAJOR) - return -EINVAL; - - /* - * Verify that the first and only mandatory parameter header is a - * Basic Flash Parameter Table header as specified in JESD216. - */ - bfpt_header = &header.bfpt_header; - if (SFDP_PARAM_HEADER_ID(bfpt_header) != SFDP_BFPT_ID || - bfpt_header->major != SFDP_JESD216_MAJOR) - return -EINVAL; - - /* - * Allocate memory then read all parameter headers with a single - * Read SFDP command. These parameter headers will actually be parsed - * twice: a first time to get the latest revision of the basic flash - * parameter table, then a second time to handle the supported optional - * tables. - * Hence we read the parameter headers once for all to reduce the - * processing time. Also we use kmalloc() instead of devm_kmalloc() - * because we don't need to keep these parameter headers: the allocated - * memory is always released with kfree() before exiting this function. - */ - if (header.nph) { - psize = header.nph * sizeof(*param_headers); - - param_headers = kmalloc(psize, GFP_KERNEL); - if (!param_headers) - return -ENOMEM; - - err = spi_nor_read_sfdp(nor, sizeof(header), - psize, param_headers); - if (err < 0) { - dev_dbg(dev, "failed to read SFDP parameter headers\n"); - goto exit; - } - } - - /* - * Check other parameter headers to get the latest revision of - * the basic flash parameter table. - */ - for (i = 0; i < header.nph; i++) { - param_header = ¶m_headers[i]; - - if (SFDP_PARAM_HEADER_ID(param_header) == SFDP_BFPT_ID && - param_header->major == SFDP_JESD216_MAJOR && - (param_header->minor > bfpt_header->minor || - (param_header->minor == bfpt_header->minor && - param_header->length > bfpt_header->length))) - bfpt_header = param_header; - } - - err = spi_nor_parse_bfpt(nor, bfpt_header, params); - if (err) - goto exit; - - /* Parse optional parameter tables. */ - for (i = 0; i < header.nph; i++) { - param_header = ¶m_headers[i]; - - switch (SFDP_PARAM_HEADER_ID(param_header)) { - case SFDP_SECTOR_MAP_ID: - err = spi_nor_parse_smpt(nor, param_header, params); - break; - - case SFDP_4BAIT_ID: - err = spi_nor_parse_4bait(nor, param_header, params); - break; - - default: - break; - } - - if (err) { - dev_warn(dev, "Failed to parse optional parameter table: %04x\n", - SFDP_PARAM_HEADER_ID(param_header)); - /* - * Let's not drop all information we extracted so far - * if optional table parsers fail. In case of failing, - * each optional parser is responsible to roll back to - * the previously known spi_nor data. - */ - err = 0; - } - } - -exit: - kfree(param_headers); - return err; -} - -static int spi_nor_select_read(struct spi_nor *nor, - u32 shared_hwcaps) -{ - int cmd, best_match = fls(shared_hwcaps & SNOR_HWCAPS_READ_MASK) - 1; - const struct spi_nor_read_command *read; - - if (best_match < 0) - return -EINVAL; - - cmd = spi_nor_hwcaps_read2cmd(BIT(best_match)); - if (cmd < 0) - return -EINVAL; - - read = &nor->params.reads[cmd]; - nor->read_opcode = read->opcode; - nor->read_proto = read->proto; - - /* - * In the spi-nor framework, we don't need to make the difference - * between mode clock cycles and wait state clock cycles. - * Indeed, the value of the mode clock cycles is used by a QSPI - * flash memory to know whether it should enter or leave its 0-4-4 - * (Continuous Read / XIP) mode. - * eXecution In Place is out of the scope of the mtd sub-system. - * Hence we choose to merge both mode and wait state clock cycles - * into the so called dummy clock cycles. - */ - nor->read_dummy = read->num_mode_clocks + read->num_wait_states; - return 0; -} - -static int spi_nor_select_pp(struct spi_nor *nor, - u32 shared_hwcaps) -{ - int cmd, best_match = fls(shared_hwcaps & SNOR_HWCAPS_PP_MASK) - 1; - const struct spi_nor_pp_command *pp; - - if (best_match < 0) - return -EINVAL; - - cmd = spi_nor_hwcaps_pp2cmd(BIT(best_match)); - if (cmd < 0) - return -EINVAL; - - pp = &nor->params.page_programs[cmd]; - nor->program_opcode = pp->opcode; - nor->write_proto = pp->proto; - return 0; -} - -/** - * spi_nor_select_uniform_erase() - select optimum uniform erase type - * @map: the erase map of the SPI NOR - * @wanted_size: the erase type size to search for. Contains the value of - * info->sector_size or of the "small sector" size in case - * CONFIG_MTD_SPI_NOR_USE_4K_SECTORS is defined. - * - * Once the optimum uniform sector erase command is found, disable all the - * other. - * - * Return: pointer to erase type on success, NULL otherwise. - */ -static const struct spi_nor_erase_type * -spi_nor_select_uniform_erase(struct spi_nor_erase_map *map, - const u32 wanted_size) -{ - const struct spi_nor_erase_type *tested_erase, *erase = NULL; - int i; - u8 uniform_erase_type = map->uniform_erase_type; - - for (i = SNOR_ERASE_TYPE_MAX - 1; i >= 0; i--) { - if (!(uniform_erase_type & BIT(i))) - continue; - - tested_erase = &map->erase_type[i]; - - /* - * If the current erase size is the one, stop here: - * we have found the right uniform Sector Erase command. - */ - if (tested_erase->size == wanted_size) { - erase = tested_erase; - break; - } - - /* - * Otherwise, the current erase size is still a valid canditate. - * Select the biggest valid candidate. - */ - if (!erase && tested_erase->size) - erase = tested_erase; - /* keep iterating to find the wanted_size */ - } - - if (!erase) - return NULL; - - /* Disable all other Sector Erase commands. */ - map->uniform_erase_type &= ~SNOR_ERASE_TYPE_MASK; - map->uniform_erase_type |= BIT(erase - map->erase_type); - return erase; -} - -static int spi_nor_select_erase(struct spi_nor *nor) -{ - struct spi_nor_erase_map *map = &nor->params.erase_map; - const struct spi_nor_erase_type *erase = NULL; - struct mtd_info *mtd = &nor->mtd; - u32 wanted_size = nor->info->sector_size; - int i; - - /* - * The previous implementation handling Sector Erase commands assumed - * that the SPI flash memory has an uniform layout then used only one - * of the supported erase sizes for all Sector Erase commands. - * So to be backward compatible, the new implementation also tries to - * manage the SPI flash memory as uniform with a single erase sector - * size, when possible. - */ -#ifdef CONFIG_MTD_SPI_NOR_USE_4K_SECTORS - /* prefer "small sector" erase if possible */ - wanted_size = 4096u; -#endif - - if (spi_nor_has_uniform_erase(nor)) { - erase = spi_nor_select_uniform_erase(map, wanted_size); - if (!erase) - return -EINVAL; - nor->erase_opcode = erase->opcode; - mtd->erasesize = erase->size; - return 0; - } - - /* - * For non-uniform SPI flash memory, set mtd->erasesize to the - * maximum erase sector size. No need to set nor->erase_opcode. - */ - for (i = SNOR_ERASE_TYPE_MAX - 1; i >= 0; i--) { - if (map->erase_type[i].size) { - erase = &map->erase_type[i]; - break; - } - } - - if (!erase) - return -EINVAL; - - mtd->erasesize = erase->size; - return 0; -} - -static int spi_nor_default_setup(struct spi_nor *nor, - const struct spi_nor_hwcaps *hwcaps) -{ - struct spi_nor_flash_parameter *params = &nor->params; - u32 ignored_mask, shared_mask; - int err; - - /* - * Keep only the hardware capabilities supported by both the SPI - * controller and the SPI flash memory. - */ - shared_mask = hwcaps->mask & params->hwcaps.mask; - - if (nor->spimem) { - /* - * When called from spi_nor_probe(), all caps are set and we - * need to discard some of them based on what the SPI - * controller actually supports (using spi_mem_supports_op()). - */ - spi_nor_spimem_adjust_hwcaps(nor, &shared_mask); - } else { - /* - * SPI n-n-n protocols are not supported when the SPI - * controller directly implements the spi_nor interface. - * Yet another reason to switch to spi-mem. - */ - ignored_mask = SNOR_HWCAPS_X_X_X; - if (shared_mask & ignored_mask) { - dev_dbg(nor->dev, - "SPI n-n-n protocols are not supported.\n"); - shared_mask &= ~ignored_mask; - } - } - - /* Select the (Fast) Read command. */ - err = spi_nor_select_read(nor, shared_mask); - if (err) { - dev_dbg(nor->dev, - "can't select read settings supported by both the SPI controller and memory.\n"); - return err; - } - - /* Select the Page Program command. */ - err = spi_nor_select_pp(nor, shared_mask); - if (err) { - dev_dbg(nor->dev, - "can't select write settings supported by both the SPI controller and memory.\n"); - return err; - } - - /* Select the Sector Erase command. */ - err = spi_nor_select_erase(nor); - if (err) { - dev_dbg(nor->dev, - "can't select erase settings supported by both the SPI controller and memory.\n"); - return err; - } - - return 0; -} - -static int spi_nor_setup(struct spi_nor *nor, - const struct spi_nor_hwcaps *hwcaps) -{ - if (!nor->params.setup) - return 0; - - return nor->params.setup(nor, hwcaps); -} - -static void atmel_set_default_init(struct spi_nor *nor) -{ - nor->flags |= SNOR_F_HAS_LOCK; -} - -static void intel_set_default_init(struct spi_nor *nor) -{ - nor->flags |= SNOR_F_HAS_LOCK; -} - -static void issi_set_default_init(struct spi_nor *nor) -{ - nor->params.quad_enable = spi_nor_sr1_bit6_quad_enable; -} - -static void macronix_set_default_init(struct spi_nor *nor) -{ - nor->params.quad_enable = spi_nor_sr1_bit6_quad_enable; - nor->params.set_4byte = macronix_set_4byte; -} - -static void sst_set_default_init(struct spi_nor *nor) -{ - nor->flags |= SNOR_F_HAS_LOCK; -} - -static void st_micron_set_default_init(struct spi_nor *nor) -{ - nor->flags |= SNOR_F_HAS_LOCK; - nor->flags &= ~SNOR_F_HAS_16BIT_SR; - nor->params.quad_enable = NULL; - nor->params.set_4byte = st_micron_set_4byte; -} - -static void winbond_set_default_init(struct spi_nor *nor) -{ - nor->params.set_4byte = winbond_set_4byte; -} - -/** - * spi_nor_manufacturer_init_params() - Initialize the flash's parameters and - * settings based on MFR register and ->default_init() hook. - * @nor: pointer to a 'struct spi-nor'. - */ -static void spi_nor_manufacturer_init_params(struct spi_nor *nor) -{ - /* Init flash parameters based on MFR */ - switch (JEDEC_MFR(nor->info)) { - case SNOR_MFR_ATMEL: - atmel_set_default_init(nor); - break; - - case SNOR_MFR_INTEL: - intel_set_default_init(nor); - break; - - case SNOR_MFR_ISSI: - issi_set_default_init(nor); - break; - - case SNOR_MFR_MACRONIX: - macronix_set_default_init(nor); - break; - - case SNOR_MFR_ST: - case SNOR_MFR_MICRON: - st_micron_set_default_init(nor); - break; - - case SNOR_MFR_SST: - sst_set_default_init(nor); - break; - - case SNOR_MFR_WINBOND: - winbond_set_default_init(nor); - break; - - default: - break; - } - - if (nor->info->fixups && nor->info->fixups->default_init) - nor->info->fixups->default_init(nor); -} - -/** - * spi_nor_sfdp_init_params() - Initialize the flash's parameters and settings - * based on JESD216 SFDP standard. - * @nor: pointer to a 'struct spi-nor'. - * - * The method has a roll-back mechanism: in case the SFDP parsing fails, the - * legacy flash parameters and settings will be restored. - */ -static void spi_nor_sfdp_init_params(struct spi_nor *nor) -{ - struct spi_nor_flash_parameter sfdp_params; - - memcpy(&sfdp_params, &nor->params, sizeof(sfdp_params)); - - if (spi_nor_parse_sfdp(nor, &sfdp_params)) { - nor->addr_width = 0; - nor->flags &= ~SNOR_F_4B_OPCODES; - } else { - memcpy(&nor->params, &sfdp_params, sizeof(nor->params)); - } -} - -/** - * spi_nor_info_init_params() - Initialize the flash's parameters and settings - * based on nor->info data. - * @nor: pointer to a 'struct spi-nor'. - */ -static void spi_nor_info_init_params(struct spi_nor *nor) -{ - struct spi_nor_flash_parameter *params = &nor->params; - struct spi_nor_erase_map *map = ¶ms->erase_map; - const struct flash_info *info = nor->info; - struct device_node *np = spi_nor_get_flash_node(nor); - u8 i, erase_mask; - - /* Initialize legacy flash parameters and settings. */ - params->quad_enable = spi_nor_sr2_bit1_quad_enable; - params->set_4byte = spansion_set_4byte; - params->setup = spi_nor_default_setup; - /* Default to 16-bit Write Status (01h) Command */ - nor->flags |= SNOR_F_HAS_16BIT_SR; - - /* Set SPI NOR sizes. */ - params->size = (u64)info->sector_size * info->n_sectors; - params->page_size = info->page_size; - - if (!(info->flags & SPI_NOR_NO_FR)) { - /* Default to Fast Read for DT and non-DT platform devices. */ - params->hwcaps.mask |= SNOR_HWCAPS_READ_FAST; - - /* Mask out Fast Read if not requested at DT instantiation. */ - if (np && !of_property_read_bool(np, "m25p,fast-read")) - params->hwcaps.mask &= ~SNOR_HWCAPS_READ_FAST; - } - - /* (Fast) Read settings. */ - params->hwcaps.mask |= SNOR_HWCAPS_READ; - spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ], - 0, 0, SPINOR_OP_READ, - SNOR_PROTO_1_1_1); - - if (params->hwcaps.mask & SNOR_HWCAPS_READ_FAST) - spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ_FAST], - 0, 8, SPINOR_OP_READ_FAST, - SNOR_PROTO_1_1_1); - - if (info->flags & SPI_NOR_DUAL_READ) { - params->hwcaps.mask |= SNOR_HWCAPS_READ_1_1_2; - spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ_1_1_2], - 0, 8, SPINOR_OP_READ_1_1_2, - SNOR_PROTO_1_1_2); - } - - if (info->flags & SPI_NOR_QUAD_READ) { - params->hwcaps.mask |= SNOR_HWCAPS_READ_1_1_4; - spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ_1_1_4], - 0, 8, SPINOR_OP_READ_1_1_4, - SNOR_PROTO_1_1_4); - } - - if (info->flags & SPI_NOR_OCTAL_READ) { - params->hwcaps.mask |= SNOR_HWCAPS_READ_1_1_8; - spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ_1_1_8], - 0, 8, SPINOR_OP_READ_1_1_8, - SNOR_PROTO_1_1_8); - } - - /* Page Program settings. */ - params->hwcaps.mask |= SNOR_HWCAPS_PP; - spi_nor_set_pp_settings(¶ms->page_programs[SNOR_CMD_PP], - SPINOR_OP_PP, SNOR_PROTO_1_1_1); - - /* - * Sector Erase settings. Sort Erase Types in ascending order, with the - * smallest erase size starting at BIT(0). - */ - erase_mask = 0; - i = 0; - if (info->flags & SECT_4K_PMC) { - erase_mask |= BIT(i); - spi_nor_set_erase_type(&map->erase_type[i], 4096u, - SPINOR_OP_BE_4K_PMC); - i++; - } else if (info->flags & SECT_4K) { - erase_mask |= BIT(i); - spi_nor_set_erase_type(&map->erase_type[i], 4096u, - SPINOR_OP_BE_4K); - i++; - } - erase_mask |= BIT(i); - spi_nor_set_erase_type(&map->erase_type[i], info->sector_size, - SPINOR_OP_SE); - spi_nor_init_uniform_erase_map(map, erase_mask, params->size); -} - -static void spansion_post_sfdp_fixups(struct spi_nor *nor) -{ - if (nor->params.size <= SZ_16M) - return; - - nor->flags |= SNOR_F_4B_OPCODES; - /* No small sector erase for 4-byte command set */ - nor->erase_opcode = SPINOR_OP_SE; - nor->mtd.erasesize = nor->info->sector_size; -} - -static void s3an_post_sfdp_fixups(struct spi_nor *nor) -{ - nor->params.setup = s3an_nor_setup; -} - -/** - * spi_nor_post_sfdp_fixups() - Updates the flash's parameters and settings - * after SFDP has been parsed (is also called for SPI NORs that do not - * support RDSFDP). - * @nor: pointer to a 'struct spi_nor' - * - * Typically used to tweak various parameters that could not be extracted by - * other means (i.e. when information provided by the SFDP/flash_info tables - * are incomplete or wrong). - */ -static void spi_nor_post_sfdp_fixups(struct spi_nor *nor) -{ - switch (JEDEC_MFR(nor->info)) { - case SNOR_MFR_SPANSION: - spansion_post_sfdp_fixups(nor); - break; - - default: - break; - } - - if (nor->info->flags & SPI_S3AN) - s3an_post_sfdp_fixups(nor); - - if (nor->info->fixups && nor->info->fixups->post_sfdp) - nor->info->fixups->post_sfdp(nor); -} - -/** - * spi_nor_late_init_params() - Late initialization of default flash parameters. - * @nor: pointer to a 'struct spi_nor' - * - * Used to set default flash parameters and settings when the ->default_init() - * hook or the SFDP parser let voids. - */ -static void spi_nor_late_init_params(struct spi_nor *nor) -{ - /* - * NOR protection support. When locking_ops are not provided, we pick - * the default ones. - */ - if (nor->flags & SNOR_F_HAS_LOCK && !nor->params.locking_ops) - nor->params.locking_ops = &stm_locking_ops; -} - -/** - * spi_nor_init_params() - Initialize the flash's parameters and settings. - * @nor: pointer to a 'struct spi-nor'. - * - * The flash parameters and settings are initialized based on a sequence of - * calls that are ordered by priority: - * - * 1/ Default flash parameters initialization. The initializations are done - * based on nor->info data: - * spi_nor_info_init_params() - * - * which can be overwritten by: - * 2/ Manufacturer flash parameters initialization. The initializations are - * done based on MFR register, or when the decisions can not be done solely - * based on MFR, by using specific flash_info tweeks, ->default_init(): - * spi_nor_manufacturer_init_params() - * - * which can be overwritten by: - * 3/ SFDP flash parameters initialization. JESD216 SFDP is a standard and - * should be more accurate that the above. - * spi_nor_sfdp_init_params() - * - * Please note that there is a ->post_bfpt() fixup hook that can overwrite - * the flash parameters and settings immediately after parsing the Basic - * Flash Parameter Table. - * - * which can be overwritten by: - * 4/ Post SFDP flash parameters initialization. Used to tweak various - * parameters that could not be extracted by other means (i.e. when - * information provided by the SFDP/flash_info tables are incomplete or - * wrong). - * spi_nor_post_sfdp_fixups() - * - * 5/ Late default flash parameters initialization, used when the - * ->default_init() hook or the SFDP parser do not set specific params. - * spi_nor_late_init_params() - */ -static void spi_nor_init_params(struct spi_nor *nor) -{ - spi_nor_info_init_params(nor); - - spi_nor_manufacturer_init_params(nor); - - if ((nor->info->flags & (SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ)) && - !(nor->info->flags & SPI_NOR_SKIP_SFDP)) - spi_nor_sfdp_init_params(nor); - - spi_nor_post_sfdp_fixups(nor); - - spi_nor_late_init_params(nor); -} - -/** - * spi_nor_quad_enable() - enable Quad I/O if needed. - * @nor: pointer to a 'struct spi_nor' - * - * Return: 0 on success, -errno otherwise. - */ -static int spi_nor_quad_enable(struct spi_nor *nor) -{ - if (!nor->params.quad_enable) - return 0; - - if (!(spi_nor_get_protocol_width(nor->read_proto) == 4 || - spi_nor_get_protocol_width(nor->write_proto) == 4)) - return 0; - - return nor->params.quad_enable(nor); -} - -/** - * spi_nor_unlock_all() - Unlocks the entire flash memory array. - * @nor: pointer to a 'struct spi_nor'. - * - * Some SPI NOR flashes are write protected by default after a power-on reset - * cycle, in order to avoid inadvertent writes during power-up. Backward - * compatibility imposes to unlock the entire flash memory array at power-up - * by default. - */ -static int spi_nor_unlock_all(struct spi_nor *nor) -{ - if (nor->flags & SNOR_F_HAS_LOCK) - return spi_nor_unlock(&nor->mtd, 0, nor->params.size); - - return 0; -} - -static int spi_nor_init(struct spi_nor *nor) -{ - int err; - - err = spi_nor_quad_enable(nor); - if (err) { - dev_dbg(nor->dev, "quad mode not supported\n"); - return err; - } - - err = spi_nor_unlock_all(nor); - if (err) { - dev_dbg(nor->dev, "Failed to unlock the entire flash memory array\n"); - return err; - } - - if (nor->addr_width == 4 && !(nor->flags & SNOR_F_4B_OPCODES)) { - /* - * If the RESET# pin isn't hooked up properly, or the system - * otherwise doesn't perform a reset command in the boot - * sequence, it's impossible to 100% protect against unexpected - * reboots (e.g., crashes). Warn the user (or hopefully, system - * designer) that this is bad. - */ - WARN_ONCE(nor->flags & SNOR_F_BROKEN_RESET, - "enabling reset hack; may not recover from unexpected reboots\n"); - nor->params.set_4byte(nor, true); - } - - return 0; -} - -/* mtd resume handler */ -static void spi_nor_resume(struct mtd_info *mtd) -{ - struct spi_nor *nor = mtd_to_spi_nor(mtd); - struct device *dev = nor->dev; - int ret; - - /* re-initialize the nor chip */ - ret = spi_nor_init(nor); - if (ret) - dev_err(dev, "resume() failed\n"); -} - -void spi_nor_restore(struct spi_nor *nor) -{ - /* restore the addressing mode */ - if (nor->addr_width == 4 && !(nor->flags & SNOR_F_4B_OPCODES) && - nor->flags & SNOR_F_BROKEN_RESET) - nor->params.set_4byte(nor, false); -} -EXPORT_SYMBOL_GPL(spi_nor_restore); - -static const struct flash_info *spi_nor_match_id(const char *name) -{ - const struct flash_info *id = spi_nor_ids; - - while (id->name) { - if (!strcmp(name, id->name)) - return id; - id++; - } - return NULL; -} - -static int spi_nor_set_addr_width(struct spi_nor *nor) -{ - if (nor->addr_width) { - /* already configured from SFDP */ - } else if (nor->info->addr_width) { - nor->addr_width = nor->info->addr_width; - } else if (nor->mtd.size > 0x1000000) { - /* enable 4-byte addressing if the device exceeds 16MiB */ - nor->addr_width = 4; - } else { - nor->addr_width = 3; - } - - if (nor->addr_width > SPI_NOR_MAX_ADDR_WIDTH) { - dev_dbg(nor->dev, "address width is too large: %u\n", - nor->addr_width); - return -EINVAL; - } - - /* Set 4byte opcodes when possible. */ - if (nor->addr_width == 4 && nor->flags & SNOR_F_4B_OPCODES && - !(nor->flags & SNOR_F_HAS_4BAIT)) - spi_nor_set_4byte_opcodes(nor); - - return 0; -} - -static void spi_nor_debugfs_init(struct spi_nor *nor, - const struct flash_info *info) -{ - struct mtd_info *mtd = &nor->mtd; - - mtd->dbg.partname = info->name; - mtd->dbg.partid = devm_kasprintf(nor->dev, GFP_KERNEL, "spi-nor:%*phN", - info->id_len, info->id); -} - -static const struct flash_info *spi_nor_get_flash_info(struct spi_nor *nor, - const char *name) -{ - const struct flash_info *info = NULL; - - if (name) - info = spi_nor_match_id(name); - /* Try to auto-detect if chip name wasn't specified or not found */ - if (!info) - info = spi_nor_read_id(nor); - if (IS_ERR_OR_NULL(info)) - return ERR_PTR(-ENOENT); - - /* - * If caller has specified name of flash model that can normally be - * detected using JEDEC, let's verify it. - */ - if (name && info->id_len) { - const struct flash_info *jinfo; - - jinfo = spi_nor_read_id(nor); - if (IS_ERR(jinfo)) { - return jinfo; - } else if (jinfo != info) { - /* - * JEDEC knows better, so overwrite platform ID. We - * can't trust partitions any longer, but we'll let - * mtd apply them anyway, since some partitions may be - * marked read-only, and we don't want to lose that - * information, even if it's not 100% accurate. - */ - dev_warn(nor->dev, "found %s, expected %s\n", - jinfo->name, info->name); - info = jinfo; - } - } - - return info; -} - -int spi_nor_scan(struct spi_nor *nor, const char *name, - const struct spi_nor_hwcaps *hwcaps) -{ - const struct flash_info *info; - struct device *dev = nor->dev; - struct mtd_info *mtd = &nor->mtd; - struct device_node *np = spi_nor_get_flash_node(nor); - struct spi_nor_flash_parameter *params = &nor->params; - int ret; - int i; - - ret = spi_nor_check(nor); - if (ret) - return ret; - - /* Reset SPI protocol for all commands. */ - nor->reg_proto = SNOR_PROTO_1_1_1; - nor->read_proto = SNOR_PROTO_1_1_1; - nor->write_proto = SNOR_PROTO_1_1_1; - - /* - * We need the bounce buffer early to read/write registers when going - * through the spi-mem layer (buffers have to be DMA-able). - * For spi-mem drivers, we'll reallocate a new buffer if - * nor->page_size turns out to be greater than PAGE_SIZE (which - * shouldn't happen before long since NOR pages are usually less - * than 1KB) after spi_nor_scan() returns. - */ - nor->bouncebuf_size = PAGE_SIZE; - nor->bouncebuf = devm_kmalloc(dev, nor->bouncebuf_size, - GFP_KERNEL); - if (!nor->bouncebuf) - return -ENOMEM; - - info = spi_nor_get_flash_info(nor, name); - if (IS_ERR(info)) - return PTR_ERR(info); - - nor->info = info; - - spi_nor_debugfs_init(nor, info); - - mutex_init(&nor->lock); - - /* - * Make sure the XSR_RDY flag is set before calling - * spi_nor_wait_till_ready(). Xilinx S3AN share MFR - * with Atmel spi-nor - */ - if (info->flags & SPI_NOR_XSR_RDY) - nor->flags |= SNOR_F_READY_XSR_RDY; - - if (info->flags & SPI_NOR_HAS_LOCK) - nor->flags |= SNOR_F_HAS_LOCK; - - /* Init flash parameters based on flash_info struct and SFDP */ - spi_nor_init_params(nor); - - if (!mtd->name) - mtd->name = dev_name(dev); - mtd->priv = nor; - mtd->type = MTD_NORFLASH; - mtd->writesize = 1; - mtd->flags = MTD_CAP_NORFLASH; - mtd->size = params->size; - mtd->_erase = spi_nor_erase; - mtd->_read = spi_nor_read; - mtd->_resume = spi_nor_resume; - - if (nor->params.locking_ops) { - mtd->_lock = spi_nor_lock; - mtd->_unlock = spi_nor_unlock; - mtd->_is_locked = spi_nor_is_locked; - } - - /* sst nor chips use AAI word program */ - if (info->flags & SST_WRITE) - mtd->_write = sst_write; - else - mtd->_write = spi_nor_write; - - if (info->flags & USE_FSR) - nor->flags |= SNOR_F_USE_FSR; - if (info->flags & SPI_NOR_HAS_TB) { - nor->flags |= SNOR_F_HAS_SR_TB; - if (info->flags & SPI_NOR_TB_SR_BIT6) - nor->flags |= SNOR_F_HAS_SR_TB_BIT6; - } - - if (info->flags & NO_CHIP_ERASE) - nor->flags |= SNOR_F_NO_OP_CHIP_ERASE; - if (info->flags & USE_CLSR) - nor->flags |= SNOR_F_USE_CLSR; - - if (info->flags & SPI_NOR_NO_ERASE) - mtd->flags |= MTD_NO_ERASE; - - mtd->dev.parent = dev; - nor->page_size = params->page_size; - mtd->writebufsize = nor->page_size; - - if (of_property_read_bool(np, "broken-flash-reset")) - nor->flags |= SNOR_F_BROKEN_RESET; - - /* - * Configure the SPI memory: - * - select op codes for (Fast) Read, Page Program and Sector Erase. - * - set the number of dummy cycles (mode cycles + wait states). - * - set the SPI protocols for register and memory accesses. - */ - ret = spi_nor_setup(nor, hwcaps); - if (ret) - return ret; - - if (info->flags & SPI_NOR_4B_OPCODES) - nor->flags |= SNOR_F_4B_OPCODES; - - ret = spi_nor_set_addr_width(nor); - if (ret) - return ret; - - /* Send all the required SPI flash commands to initialize device */ - ret = spi_nor_init(nor); - if (ret) - return ret; - - dev_info(dev, "%s (%lld Kbytes)\n", info->name, - (long long)mtd->size >> 10); - - dev_dbg(dev, - "mtd .name = %s, .size = 0x%llx (%lldMiB), " - ".erasesize = 0x%.8x (%uKiB) .numeraseregions = %d\n", - mtd->name, (long long)mtd->size, (long long)(mtd->size >> 20), - mtd->erasesize, mtd->erasesize / 1024, mtd->numeraseregions); - - if (mtd->numeraseregions) - for (i = 0; i < mtd->numeraseregions; i++) - dev_dbg(dev, - "mtd.eraseregions[%d] = { .offset = 0x%llx, " - ".erasesize = 0x%.8x (%uKiB), " - ".numblocks = %d }\n", - i, (long long)mtd->eraseregions[i].offset, - mtd->eraseregions[i].erasesize, - mtd->eraseregions[i].erasesize / 1024, - mtd->eraseregions[i].numblocks); - return 0; -} -EXPORT_SYMBOL_GPL(spi_nor_scan); - -static int spi_nor_probe(struct spi_mem *spimem) -{ - struct spi_device *spi = spimem->spi; - struct flash_platform_data *data = dev_get_platdata(&spi->dev); - struct spi_nor *nor; - /* - * Enable all caps by default. The core will mask them after - * checking what's really supported using spi_mem_supports_op(). - */ - const struct spi_nor_hwcaps hwcaps = { .mask = SNOR_HWCAPS_ALL }; - char *flash_name; - int ret; - - nor = devm_kzalloc(&spi->dev, sizeof(*nor), GFP_KERNEL); - if (!nor) - return -ENOMEM; - - nor->spimem = spimem; - nor->dev = &spi->dev; - spi_nor_set_flash_node(nor, spi->dev.of_node); - - spi_mem_set_drvdata(spimem, nor); - - if (data && data->name) - nor->mtd.name = data->name; - - if (!nor->mtd.name) - nor->mtd.name = spi_mem_get_name(spimem); - - /* - * For some (historical?) reason many platforms provide two different - * names in flash_platform_data: "name" and "type". Quite often name is - * set to "m25p80" and then "type" provides a real chip name. - * If that's the case, respect "type" and ignore a "name". - */ - if (data && data->type) - flash_name = data->type; - else if (!strcmp(spi->modalias, "spi-nor")) - flash_name = NULL; /* auto-detect */ - else - flash_name = spi->modalias; - - ret = spi_nor_scan(nor, flash_name, &hwcaps); - if (ret) - return ret; - - /* - * None of the existing parts have > 512B pages, but let's play safe - * and add this logic so that if anyone ever adds support for such - * a NOR we don't end up with buffer overflows. - */ - if (nor->page_size > PAGE_SIZE) { - nor->bouncebuf_size = nor->page_size; - devm_kfree(nor->dev, nor->bouncebuf); - nor->bouncebuf = devm_kmalloc(nor->dev, - nor->bouncebuf_size, - GFP_KERNEL); - if (!nor->bouncebuf) - return -ENOMEM; - } - - return mtd_device_register(&nor->mtd, data ? data->parts : NULL, - data ? data->nr_parts : 0); -} - -static int spi_nor_remove(struct spi_mem *spimem) -{ - struct spi_nor *nor = spi_mem_get_drvdata(spimem); - - spi_nor_restore(nor); - - /* Clean up MTD stuff. */ - return mtd_device_unregister(&nor->mtd); -} - -static void spi_nor_shutdown(struct spi_mem *spimem) -{ - struct spi_nor *nor = spi_mem_get_drvdata(spimem); - - spi_nor_restore(nor); -} - -/* - * Do NOT add to this array without reading the following: - * - * Historically, many flash devices are bound to this driver by their name. But - * since most of these flash are compatible to some extent, and their - * differences can often be differentiated by the JEDEC read-ID command, we - * encourage new users to add support to the spi-nor library, and simply bind - * against a generic string here (e.g., "jedec,spi-nor"). - * - * Many flash names are kept here in this list (as well as in spi-nor.c) to - * keep them available as module aliases for existing platforms. - */ -static const struct spi_device_id spi_nor_dev_ids[] = { - /* - * Allow non-DT platform devices to bind to the "spi-nor" modalias, and - * hack around the fact that the SPI core does not provide uevent - * matching for .of_match_table - */ - {"spi-nor"}, - - /* - * Entries not used in DTs that should be safe to drop after replacing - * them with "spi-nor" in platform data. - */ - {"s25sl064a"}, {"w25x16"}, {"m25p10"}, {"m25px64"}, - - /* - * Entries that were used in DTs without "jedec,spi-nor" fallback and - * should be kept for backward compatibility. - */ - {"at25df321a"}, {"at25df641"}, {"at26df081a"}, - {"mx25l4005a"}, {"mx25l1606e"}, {"mx25l6405d"}, {"mx25l12805d"}, - {"mx25l25635e"},{"mx66l51235l"}, - {"n25q064"}, {"n25q128a11"}, {"n25q128a13"}, {"n25q512a"}, - {"s25fl256s1"}, {"s25fl512s"}, {"s25sl12801"}, {"s25fl008k"}, - {"s25fl064k"}, - {"sst25vf040b"},{"sst25vf016b"},{"sst25vf032b"},{"sst25wf040"}, - {"m25p40"}, {"m25p80"}, {"m25p16"}, {"m25p32"}, - {"m25p64"}, {"m25p128"}, - {"w25x80"}, {"w25x32"}, {"w25q32"}, {"w25q32dw"}, - {"w25q80bl"}, {"w25q128"}, {"w25q256"}, - - /* Flashes that can't be detected using JEDEC */ - {"m25p05-nonjedec"}, {"m25p10-nonjedec"}, {"m25p20-nonjedec"}, - {"m25p40-nonjedec"}, {"m25p80-nonjedec"}, {"m25p16-nonjedec"}, - {"m25p32-nonjedec"}, {"m25p64-nonjedec"}, {"m25p128-nonjedec"}, - - /* Everspin MRAMs (non-JEDEC) */ - { "mr25h128" }, /* 128 Kib, 40 MHz */ - { "mr25h256" }, /* 256 Kib, 40 MHz */ - { "mr25h10" }, /* 1 Mib, 40 MHz */ - { "mr25h40" }, /* 4 Mib, 40 MHz */ - - { }, -}; -MODULE_DEVICE_TABLE(spi, spi_nor_dev_ids); - -static const struct of_device_id spi_nor_of_table[] = { - /* - * Generic compatibility for SPI NOR that can be identified by the - * JEDEC READ ID opcode (0x9F). Use this, if possible. - */ - { .compatible = "jedec,spi-nor" }, - { /* sentinel */ }, -}; -MODULE_DEVICE_TABLE(of, spi_nor_of_table); - -/* - * REVISIT: many of these chips have deep power-down modes, which - * should clearly be entered on suspend() to minimize power use. - * And also when they're otherwise idle... - */ -static struct spi_mem_driver spi_nor_driver = { - .spidrv = { - .driver = { - .name = "spi-nor", - .of_match_table = spi_nor_of_table, - }, - .id_table = spi_nor_dev_ids, - }, - .probe = spi_nor_probe, - .remove = spi_nor_remove, - .shutdown = spi_nor_shutdown, -}; -module_spi_mem_driver(spi_nor_driver); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Huang Shijie "); -MODULE_AUTHOR("Mike Lavender"); -MODULE_DESCRIPTION("framework for SPI NOR"); diff --git a/drivers/mtd/spi-nor/sst.c b/drivers/mtd/spi-nor/sst.c new file mode 100644 index 000000000000..e0af6d25d573 --- /dev/null +++ b/drivers/mtd/spi-nor/sst.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info sst_parts[] = { + /* SST -- large erase sizes are "overlays", "sectors" are 4K */ + { "sst25vf040b", INFO(0xbf258d, 0, 64 * 1024, 8, + SECT_4K | SST_WRITE) }, + { "sst25vf080b", INFO(0xbf258e, 0, 64 * 1024, 16, + SECT_4K | SST_WRITE) }, + { "sst25vf016b", INFO(0xbf2541, 0, 64 * 1024, 32, + SECT_4K | SST_WRITE) }, + { "sst25vf032b", INFO(0xbf254a, 0, 64 * 1024, 64, + SECT_4K | SST_WRITE) }, + { "sst25vf064c", INFO(0xbf254b, 0, 64 * 1024, 128, SECT_4K) }, + { "sst25wf512", INFO(0xbf2501, 0, 64 * 1024, 1, + SECT_4K | SST_WRITE) }, + { "sst25wf010", INFO(0xbf2502, 0, 64 * 1024, 2, + SECT_4K | SST_WRITE) }, + { "sst25wf020", INFO(0xbf2503, 0, 64 * 1024, 4, + SECT_4K | SST_WRITE) }, + { "sst25wf020a", INFO(0x621612, 0, 64 * 1024, 4, SECT_4K) }, + { "sst25wf040b", INFO(0x621613, 0, 64 * 1024, 8, SECT_4K) }, + { "sst25wf040", INFO(0xbf2504, 0, 64 * 1024, 8, + SECT_4K | SST_WRITE) }, + { "sst25wf080", INFO(0xbf2505, 0, 64 * 1024, 16, + SECT_4K | SST_WRITE) }, + { "sst26wf016b", INFO(0xbf2651, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ) }, + { "sst26vf016b", INFO(0xbf2641, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_DUAL_READ) }, + { "sst26vf064b", INFO(0xbf2643, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ) }, +}; + +static int sst_write(struct mtd_info *mtd, loff_t to, size_t len, + size_t *retlen, const u_char *buf) +{ + struct spi_nor *nor = mtd_to_spi_nor(mtd); + size_t actual = 0; + int ret; + + dev_dbg(nor->dev, "to 0x%08x, len %zd\n", (u32)to, len); + + ret = spi_nor_lock_and_prep(nor); + if (ret) + return ret; + + ret = spi_nor_write_enable(nor); + if (ret) + goto out; + + nor->sst_write_second = false; + + /* Start write from odd address. */ + if (to % 2) { + nor->program_opcode = SPINOR_OP_BP; + + /* write one byte. */ + ret = spi_nor_write_data(nor, to, 1, buf); + if (ret < 0) + goto out; + WARN(ret != 1, "While writing 1 byte written %i bytes\n", ret); + ret = spi_nor_wait_till_ready(nor); + if (ret) + goto out; + + to++; + actual++; + } + + /* Write out most of the data here. */ + for (; actual < len - 1; actual += 2) { + nor->program_opcode = SPINOR_OP_AAI_WP; + + /* write two bytes. */ + ret = spi_nor_write_data(nor, to, 2, buf + actual); + if (ret < 0) + goto out; + WARN(ret != 2, "While writing 2 bytes written %i bytes\n", ret); + ret = spi_nor_wait_till_ready(nor); + if (ret) + goto out; + to += 2; + nor->sst_write_second = true; + } + nor->sst_write_second = false; + + ret = spi_nor_write_disable(nor); + if (ret) + goto out; + + ret = spi_nor_wait_till_ready(nor); + if (ret) + goto out; + + /* Write out trailing byte if it exists. */ + if (actual != len) { + ret = spi_nor_write_enable(nor); + if (ret) + goto out; + + nor->program_opcode = SPINOR_OP_BP; + ret = spi_nor_write_data(nor, to, 1, buf + actual); + if (ret < 0) + goto out; + WARN(ret != 1, "While writing 1 byte written %i bytes\n", ret); + ret = spi_nor_wait_till_ready(nor); + if (ret) + goto out; + + actual += 1; + + ret = spi_nor_write_disable(nor); + } +out: + *retlen += actual; + spi_nor_unlock_and_unprep(nor); + return ret; +} + +static void sst_default_init(struct spi_nor *nor) +{ + nor->flags |= SNOR_F_HAS_LOCK; +} + +static void sst_post_sfdp_fixups(struct spi_nor *nor) +{ + if (nor->info->flags & SST_WRITE) + nor->mtd._write = sst_write; +} + +static const struct spi_nor_fixups sst_fixups = { + .default_init = sst_default_init, + .post_sfdp = sst_post_sfdp_fixups, +}; + +const struct spi_nor_manufacturer spi_nor_sst = { + .name = "sst", + .parts = sst_parts, + .nparts = ARRAY_SIZE(sst_parts), + .fixups = &sst_fixups, +}; diff --git a/drivers/mtd/spi-nor/winbond.c b/drivers/mtd/spi-nor/winbond.c new file mode 100644 index 000000000000..17deabad57e1 --- /dev/null +++ b/drivers/mtd/spi-nor/winbond.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info winbond_parts[] = { + /* Winbond -- w25x "blocks" are 64K, "sectors" are 4KiB */ + { "w25x05", INFO(0xef3010, 0, 64 * 1024, 1, SECT_4K) }, + { "w25x10", INFO(0xef3011, 0, 64 * 1024, 2, SECT_4K) }, + { "w25x20", INFO(0xef3012, 0, 64 * 1024, 4, SECT_4K) }, + { "w25x40", INFO(0xef3013, 0, 64 * 1024, 8, SECT_4K) }, + { "w25x80", INFO(0xef3014, 0, 64 * 1024, 16, SECT_4K) }, + { "w25x16", INFO(0xef3015, 0, 64 * 1024, 32, SECT_4K) }, + { "w25q16dw", INFO(0xef6015, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "w25x32", INFO(0xef3016, 0, 64 * 1024, 64, SECT_4K) }, + { "w25q16jv-im/jm", INFO(0xef7015, 0, 64 * 1024, 32, + SECT_4K | SPI_NOR_DUAL_READ | + SPI_NOR_QUAD_READ | SPI_NOR_HAS_LOCK | + SPI_NOR_HAS_TB) }, + { "w25q20cl", INFO(0xef4012, 0, 64 * 1024, 4, SECT_4K) }, + { "w25q20bw", INFO(0xef5012, 0, 64 * 1024, 4, SECT_4K) }, + { "w25q20ew", INFO(0xef6012, 0, 64 * 1024, 4, SECT_4K) }, + { "w25q32", INFO(0xef4016, 0, 64 * 1024, 64, SECT_4K) }, + { "w25q32dw", INFO(0xef6016, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "w25q32jv", INFO(0xef7016, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) + }, + { "w25q32jwm", INFO(0xef8016, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "w25x64", INFO(0xef3017, 0, 64 * 1024, 128, SECT_4K) }, + { "w25q64", INFO(0xef4017, 0, 64 * 1024, 128, SECT_4K) }, + { "w25q64dw", INFO(0xef6017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "w25q128fw", INFO(0xef6018, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "w25q128jv", INFO(0xef7018, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB) }, + { "w25q80", INFO(0xef5014, 0, 64 * 1024, 16, SECT_4K) }, + { "w25q80bl", INFO(0xef4014, 0, 64 * 1024, 16, SECT_4K) }, + { "w25q128", INFO(0xef4018, 0, 64 * 1024, 256, SECT_4K) }, + { "w25q256", INFO(0xef4019, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | + SPI_NOR_4B_OPCODES) }, + { "w25q256jvm", INFO(0xef7019, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "w25q256jw", INFO(0xef6019, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "w25m512jv", INFO(0xef7119, 0, 64 * 1024, 1024, + SECT_4K | SPI_NOR_QUAD_READ | SPI_NOR_DUAL_READ) }, +}; + +/** + * winbond_set_4byte_addr_mode() - Set 4-byte address mode for Winbond flashes. + * @nor: pointer to 'struct spi_nor'. + * @enable: true to enter the 4-byte address mode, false to exit the 4-byte + * address mode. + * + * Return: 0 on success, -errno otherwise. + */ +static int winbond_set_4byte_addr_mode(struct spi_nor *nor, bool enable) +{ + int ret; + + ret = spi_nor_set_4byte_addr_mode(nor, enable); + if (ret || enable) + return ret; + + /* + * On Winbond W25Q256FV, leaving 4byte mode causes the Extended Address + * Register to be set to 1, so all 3-byte-address reads come from the + * second 16M. We must clear the register to enable normal behavior. + */ + ret = spi_nor_write_enable(nor); + if (ret) + return ret; + + ret = spi_nor_write_ear(nor, 0); + if (ret) + return ret; + + return spi_nor_write_disable(nor); +} + +static void winbond_default_init(struct spi_nor *nor) +{ + nor->params->set_4byte_addr_mode = winbond_set_4byte_addr_mode; +} + +static const struct spi_nor_fixups winbond_fixups = { + .default_init = winbond_default_init, +}; + +const struct spi_nor_manufacturer spi_nor_winbond = { + .name = "winbond", + .parts = winbond_parts, + .nparts = ARRAY_SIZE(winbond_parts), + .fixups = &winbond_fixups, +}; diff --git a/drivers/mtd/spi-nor/xilinx.c b/drivers/mtd/spi-nor/xilinx.c new file mode 100644 index 000000000000..1138bdbf4199 --- /dev/null +++ b/drivers/mtd/spi-nor/xilinx.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info xilinx_parts[] = { + /* Xilinx S3AN Internal Flash */ + { "3S50AN", S3AN_INFO(0x1f2200, 64, 264) }, + { "3S200AN", S3AN_INFO(0x1f2400, 256, 264) }, + { "3S400AN", S3AN_INFO(0x1f2400, 256, 264) }, + { "3S700AN", S3AN_INFO(0x1f2500, 512, 264) }, + { "3S1400AN", S3AN_INFO(0x1f2600, 512, 528) }, +}; + +/* + * This code converts an address to the Default Address Mode, that has non + * power of two page sizes. We must support this mode because it is the default + * mode supported by Xilinx tools, it can access the whole flash area and + * changing over to the Power-of-two mode is irreversible and corrupts the + * original data. + * Addr can safely be unsigned int, the biggest S3AN device is smaller than + * 4 MiB. + */ +static u32 s3an_convert_addr(struct spi_nor *nor, u32 addr) +{ + u32 offset, page; + + offset = addr % nor->page_size; + page = addr / nor->page_size; + page <<= (nor->page_size > 512) ? 10 : 9; + + return page | offset; +} + +static int xilinx_nor_setup(struct spi_nor *nor, + const struct spi_nor_hwcaps *hwcaps) +{ + int ret; + + ret = spi_nor_xread_sr(nor, nor->bouncebuf); + if (ret) + return ret; + + nor->erase_opcode = SPINOR_OP_XSE; + nor->program_opcode = SPINOR_OP_XPP; + nor->read_opcode = SPINOR_OP_READ; + nor->flags |= SNOR_F_NO_OP_CHIP_ERASE; + + /* + * This flashes have a page size of 264 or 528 bytes (known as + * Default addressing mode). It can be changed to a more standard + * Power of two mode where the page size is 256/512. This comes + * with a price: there is 3% less of space, the data is corrupted + * and the page size cannot be changed back to default addressing + * mode. + * + * The current addressing mode can be read from the XRDSR register + * and should not be changed, because is a destructive operation. + */ + if (nor->bouncebuf[0] & XSR_PAGESIZE) { + /* Flash in Power of 2 mode */ + nor->page_size = (nor->page_size == 264) ? 256 : 512; + nor->mtd.writebufsize = nor->page_size; + nor->mtd.size = 8 * nor->page_size * nor->info->n_sectors; + nor->mtd.erasesize = 8 * nor->page_size; + } else { + /* Flash in Default addressing mode */ + nor->params->convert_addr = s3an_convert_addr; + nor->mtd.erasesize = nor->info->sector_size; + } + + return 0; +} + +static void xilinx_post_sfdp_fixups(struct spi_nor *nor) +{ + nor->params->setup = xilinx_nor_setup; +} + +static const struct spi_nor_fixups xilinx_fixups = { + .post_sfdp = xilinx_post_sfdp_fixups, +}; + +const struct spi_nor_manufacturer spi_nor_xilinx = { + .name = "xilinx", + .parts = xilinx_parts, + .nparts = ARRAY_SIZE(xilinx_parts), + .fixups = &xilinx_fixups, +}; diff --git a/drivers/mtd/spi-nor/xmc.c b/drivers/mtd/spi-nor/xmc.c new file mode 100644 index 000000000000..2c7773b68993 --- /dev/null +++ b/drivers/mtd/spi-nor/xmc.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005, Intec Automation Inc. + * Copyright (C) 2014, Freescale Semiconductor, Inc. + */ + +#include + +#include "core.h" + +static const struct flash_info xmc_parts[] = { + /* XMC (Wuhan Xinxin Semiconductor Manufacturing Corp.) */ + { "XM25QH64A", INFO(0x207017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "XM25QH128A", INFO(0x207018, 0, 64 * 1024, 256, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, +}; + +const struct spi_nor_manufacturer spi_nor_xmc = { + .name = "xmc", + .parts = xmc_parts, + .nparts = ARRAY_SIZE(xmc_parts), +}; diff --git a/drivers/mtd/ubi/attach.c b/drivers/mtd/ubi/attach.c index ea7440ac913b..ae5abe492b52 100644 --- a/drivers/mtd/ubi/attach.c +++ b/drivers/mtd/ubi/attach.c @@ -1059,7 +1059,7 @@ static int scan_peb(struct ubi_device *ubi, struct ubi_attach_info *ai, * be a result of power cut during erasure. */ ai->maybe_bad_peb_count += 1; - /* fall through */ + fallthrough; case UBI_IO_BAD_HDR: /* * If we're facing a bad VID header we have to drop *all* diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c index 2f93c25bbaee..12c02342149c 100644 --- a/drivers/mtd/ubi/build.c +++ b/drivers/mtd/ubi/build.c @@ -1342,10 +1342,10 @@ static int bytes_str_to_int(const char *str) switch (*endp) { case 'G': result *= 1024; - /* fall through */ + fallthrough; case 'M': result *= 1024; - /* fall through */ + fallthrough; case 'K': result *= 1024; if (endp[1] == 'i' && endp[2] == 'B') diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig index df0ef69dd474..834c59950d1c 100644 --- a/drivers/pinctrl/Kconfig +++ b/drivers/pinctrl/Kconfig @@ -126,6 +126,18 @@ config PINCTRL_DA850_PUPD Driver for TI DA850/OMAP-L138/AM18XX pinconf. Used to control pullup/pulldown pin groups. +config PINCTRL_DA9062 + tristate "Dialog Semiconductor DA9062 PMIC pinctrl and GPIO Support" + depends on MFD_DA9062 + select GPIOLIB + help + The Dialog DA9062 PMIC provides multiple GPIOs that can be muxed for + different functions. This driver bundles a pinctrl driver to select the + function muxing and a GPIO driver to handle the GPIO when the GPIO + function is selected. + + Say yes to enable pinctrl and GPIO support for the DA9062 PMIC. + config PINCTRL_DIGICOLOR bool depends on OF && (ARCH_DIGICOLOR || COMPILE_TEST) diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile index 879f312bfb75..0b36a1cfca8a 100644 --- a/drivers/pinctrl/Makefile +++ b/drivers/pinctrl/Makefile @@ -16,6 +16,7 @@ obj-$(CONFIG_PINCTRL_AT91PIO4) += pinctrl-at91-pio4.o obj-$(CONFIG_PINCTRL_AMD) += pinctrl-amd.o obj-$(CONFIG_PINCTRL_BM1880) += pinctrl-bm1880.o obj-$(CONFIG_PINCTRL_DA850_PUPD) += pinctrl-da850-pupd.o +obj-$(CONFIG_PINCTRL_DA9062) += pinctrl-da9062.o obj-$(CONFIG_PINCTRL_DIGICOLOR) += pinctrl-digicolor.o obj-$(CONFIG_PINCTRL_FALCON) += pinctrl-falcon.o obj-$(CONFIG_PINCTRL_GEMINI) += pinctrl-gemini.o diff --git a/drivers/pinctrl/actions/pinctrl-s700.c b/drivers/pinctrl/actions/pinctrl-s700.c index 771d6fd50b45..47a4ccd9fed4 100644 --- a/drivers/pinctrl/actions/pinctrl-s700.c +++ b/drivers/pinctrl/actions/pinctrl-s700.c @@ -1125,317 +1125,317 @@ static const struct owl_pingroup s700_groups[] = { }; static const char * const nor_groups[] = { - "lcd0_d18", - "i2s_d0", - "i2s0_pcm0", - "i2s1_pcm0", - "i2s_d1", - "ks_in2", - "ks_in1", - "ks_in0", - "ks_in3", - "ks_out0", - "ks_out1", - "ks_out2", - "lcd0_d2", - "lvds_ee_pn", - "uart2_rx_tx", - "spi0_i2c_pcm", - "lvds_e_pn", - "sd0_d0", - "sd0_d1", - "sd0_d2_d3", - "sd1_d0_d3", - "sd0_cmd", - "sd1_cmd", - "sens0_ckout", - "sen0_pclk", + "lcd0_d18_mfp", + "i2s_d0_mfp", + "i2s0_pcm0_mfp", + "i2s1_pcm0_mfp", + "i2s_d1_mfp", + "ks_in2_mfp", + "ks_in1_mfp", + "ks_in0_mfp", + "ks_in3_mfp", + "ks_out0_mfp", + "ks_out1_mfp", + "ks_out2_mfp", + "lcd0_d2_mfp", + "lvds_ee_pn_mfp", + "uart2_rx_tx_mfp", + "spi0_i2c_pcm_mfp", + "lvds_e_pn_mfp", + "sd0_d0_mfp", + "sd0_d1_mfp", + "sd0_d2_d3_mfp", + "sd1_d0_d3_mfp", + "sd0_cmd_mfp", + "sd1_cmd_mfp", + "sens0_ckout_mfp", + "sen0_pclk_mfp", }; static const char * const eth_rmii_groups[] = { - "rgmii_txd23", - "rgmii_rxd2", - "rgmii_rxd3", - "rgmii_txd01", - "rgmii_txd0", - "rgmii_txd1", - "rgmii_txen", - "rgmii_rxen", - "rgmii_rxd1", - "rgmii_rxd0", - "rgmii_ref_clk", + "rgmii_txd23_mfp", + "rgmii_rxd2_mfp", + "rgmii_rxd3_mfp", + "rgmii_txd01_mfp", + "rgmii_txd0_mfp", + "rgmii_txd1_mfp", + "rgmii_txen_mfp", + "rgmii_rxen_mfp", + "rgmii_rxd1_mfp", + "rgmii_rxd0_mfp", + "rgmii_ref_clk_mfp", "eth_smi_dummy", }; static const char * const eth_smii_groups[] = { - "rgmii_txd0", - "rgmii_txd1", - "rgmii_rxd0", - "rgmii_rxd1", - "rgmii_ref_clk", + "rgmii_txd0_mfp", + "rgmii_txd1_mfp", + "rgmii_rxd0_mfp", + "rgmii_rxd1_mfp", + "rgmii_ref_clk_mfp", "eth_smi_dummy", }; static const char * const spi0_groups[] = { - "dsi_dn0", - "dsi_dp2", - "dsi_dp0", - "uart2_rx_tx", - "spi0_i2c_pcm", - "dsi_dn2", + "dsi_dn0_mfp", + "dsi_dp2_mfp", + "dsi_dp0_mfp", + "uart2_rx_tx_mfp", + "spi0_i2c_pcm_mfp", + "dsi_dn2_mfp", }; static const char * const spi1_groups[] = { - "uart0_rx", - "uart0_tx", + "uart0_rx_mfp", + "uart0_tx_mfp", "i2c0_mfp", }; static const char * const spi2_groups[] = { - "rgmii_txd01", - "rgmii_txd0", - "rgmii_txd1", - "rgmii_ref_clk", - "dnand_acle_ce0", + "rgmii_txd01_mfp", + "rgmii_txd0_mfp", + "rgmii_txd1_mfp", + "rgmii_ref_clk_mfp", + "dnand_acle_ce0_mfp", }; static const char * const spi3_groups[] = { - "rgmii_txen", - "rgmii_rxen", - "rgmii_rxd1", - "rgmii_rxd0", + "rgmii_txen_mfp", + "rgmii_rxen_mfp", + "rgmii_rxd1_mfp", + "rgmii_rxd0_mfp", }; static const char * const sens0_groups[] = { - "csi_cn_cp", - "sens0_ckout", - "csi_dn_dp", - "sen0_pclk", + "csi_cn_cp_mfp", + "sens0_ckout_mfp", + "csi_dn_dp_mfp", + "sen0_pclk_mfp", }; static const char * const sens1_groups[] = { - "lcd0_d18", - "ks_in2", - "ks_in1", - "ks_in0", - "ks_in3", - "ks_out0", - "ks_out1", - "ks_out2", - "sens0_ckout", - "pcm1_in", - "pcm1_clk", - "pcm1_sync", - "pcm1_out", + "lcd0_d18_mfp", + "ks_in2_mfp", + "ks_in1_mfp", + "ks_in0_mfp", + "ks_in3_mfp", + "ks_out0_mfp", + "ks_out1_mfp", + "ks_out2_mfp", + "sens0_ckout_mfp", + "pcm1_in_mfp", + "pcm1_clk_mfp", + "pcm1_sync_mfp", + "pcm1_out_mfp", }; static const char * const uart0_groups[] = { - "uart2_rtsb", - "uart2_ctsb", - "uart0_rx", - "uart0_tx", + "uart2_rtsb_mfp", + "uart2_ctsb_mfp", + "uart0_rx_mfp", + "uart0_tx_mfp", }; static const char * const uart1_groups[] = { - "sd0_d2_d3", + "sd0_d2_d3_mfp", "i2c0_mfp", }; static const char * const uart2_groups[] = { - "rgmii_txen", - "rgmii_rxen", - "rgmii_rxd1", - "rgmii_rxd0", - "dsi_dn0", - "dsi_dp2", - "dsi_dp0", - "uart2_rx_tx", - "dsi_dn2", - "uart2_rtsb", - "uart2_ctsb", - "sd0_d0", - "sd0_d1", - "sd0_d2_d3", - "uart0_rx", - "uart0_tx", + "rgmii_txen_mfp", + "rgmii_rxen_mfp", + "rgmii_rxd1_mfp", + "rgmii_rxd0_mfp", + "dsi_dn0_mfp", + "dsi_dp2_mfp", + "dsi_dp0_mfp", + "uart2_rx_tx_mfp", + "dsi_dn2_mfp", + "uart2_rtsb_mfp", + "uart2_ctsb_mfp", + "sd0_d0_mfp", + "sd0_d1_mfp", + "sd0_d2_d3_mfp", + "uart0_rx_mfp", + "uart0_tx_mfp", "i2c0_mfp", "uart2_dummy" }; static const char * const uart3_groups[] = { - "rgmii_txd23", - "rgmii_rxd2", - "rgmii_rxd3", - "uart3_rtsb", - "uart3_ctsb", + "rgmii_txd23_mfp", + "rgmii_rxd2_mfp", + "rgmii_rxd3_mfp", + "uart3_rtsb_mfp", + "uart3_ctsb_mfp", "uart3_dummy" }; static const char * const uart4_groups[] = { - "rgmii_txd01", - "rgmii_ref_clk", - "ks_out0", - "ks_out1", + "rgmii_txd01_mfp", + "rgmii_ref_clk_mfp", + "ks_out0_mfp", + "ks_out1_mfp", }; static const char * const uart5_groups[] = { - "rgmii_rxd1", - "rgmii_rxd0", - "ks_out0", - "ks_out2", - "uart3_rtsb", - "uart3_ctsb", - "sd0_d0", - "sd0_d1", + "rgmii_rxd1_mfp", + "rgmii_rxd0_mfp", + "ks_out0_mfp", + "ks_out2_mfp", + "uart3_rtsb_mfp", + "uart3_ctsb_mfp", + "sd0_d0_mfp", + "sd0_d1_mfp", }; static const char * const uart6_groups[] = { - "rgmii_txd0", - "rgmii_txd1", + "rgmii_txd0_mfp", + "rgmii_txd1_mfp", }; static const char * const i2s0_groups[] = { - "i2s_d0", - "i2s_pcm1", - "i2s0_pcm0", + "i2s_d0_mfp", + "i2s_pcm1_mfp", + "i2s0_pcm0_mfp", }; static const char * const i2s1_groups[] = { - "i2s1_pcm0", - "i2s_d1", + "i2s1_pcm0_mfp", + "i2s_d1_mfp", "i2s1_dummy", - "spi0_i2c_pcm", - "uart0_rx", - "uart0_tx", + "spi0_i2c_pcm_mfp", + "uart0_rx_mfp", + "uart0_tx_mfp", }; static const char * const pcm1_groups[] = { - "i2s_pcm1", - "spi0_i2c_pcm", - "uart0_rx", - "uart0_tx", - "pcm1_in", - "pcm1_clk", - "pcm1_sync", - "pcm1_out", + "i2s_pcm1_mfp", + "spi0_i2c_pcm_mfp", + "uart0_rx_mfp", + "uart0_tx_mfp", + "pcm1_in_mfp", + "pcm1_clk_mfp", + "pcm1_sync_mfp", + "pcm1_out_mfp", }; static const char * const pcm0_groups[] = { - "i2s0_pcm0", - "i2s1_pcm0", - "uart2_rx_tx", - "spi0_i2c_pcm", + "i2s0_pcm0_mfp", + "i2s1_pcm0_mfp", + "uart2_rx_tx_mfp", + "spi0_i2c_pcm_mfp", }; static const char * const ks_groups[] = { - "ks_in2", - "ks_in1", - "ks_in0", - "ks_in3", - "ks_out0", - "ks_out1", - "ks_out2", + "ks_in2_mfp", + "ks_in1_mfp", + "ks_in0_mfp", + "ks_in3_mfp", + "ks_out0_mfp", + "ks_out1_mfp", + "ks_out2_mfp", }; static const char * const jtag_groups[] = { - "ks_in2", - "ks_in1", - "ks_in0", - "ks_in3", - "ks_out1", - "sd0_d0", - "sd0_d2_d3", - "sd0_cmd", - "sd0_clk", + "ks_in2_mfp", + "ks_in1_mfp", + "ks_in0_mfp", + "ks_in3_mfp", + "ks_out1_mfp", + "sd0_d0_mfp", + "sd0_d2_d3_mfp", + "sd0_cmd_mfp", + "sd0_clk_mfp", }; static const char * const pwm0_groups[] = { - "rgmii_rxd2", - "rgmii_txen", - "ks_in2", - "sen0_pclk", + "rgmii_rxd2_mfp", + "rgmii_txen_mfp", + "ks_in2_mfp", + "sen0_pclk_mfp", }; static const char * const pwm1_groups[] = { - "rgmii_rxen", - "ks_in1", - "ks_in3", - "sens0_ckout", + "rgmii_rxen_mfp", + "ks_in1_mfp", + "ks_in3_mfp", + "sens0_ckout_mfp", }; static const char * const pwm2_groups[] = { - "lcd0_d18", - "rgmii_rxd3", - "rgmii_rxd1", - "ks_out0", - "ks_out2", + "lcd0_d18_mfp", + "rgmii_rxd3_mfp", + "rgmii_rxd1_mfp", + "ks_out0_mfp", + "ks_out2_mfp", }; static const char * const pwm3_groups[] = { - "rgmii_rxd0", - "ks_out1", - "lcd0_d2", + "rgmii_rxd0_mfp", + "ks_out1_mfp", + "lcd0_d2_mfp", }; static const char * const pwm4_groups[] = { - "lcd0_d18", - "rgmii_txd01", - "rgmii_txd0", - "ks_in0", - "pcm1_in", - "nand_ceb3", + "lcd0_d18_mfp", + "rgmii_txd01_mfp", + "rgmii_txd0_mfp", + "ks_in0_mfp", + "pcm1_in_mfp", + "nand_ceb3_mfp", }; static const char * const pwm5_groups[] = { - "rgmii_txd1", - "ks_in1", - "pcm1_clk", - "nand_ceb2", + "rgmii_txd1_mfp", + "ks_in1_mfp", + "pcm1_clk_mfp", + "nand_ceb2_mfp", }; static const char * const p0_groups[] = { - "ks_in2", - "ks_in0", + "ks_in2_mfp", + "ks_in0_mfp", }; static const char * const sd0_groups[] = { - "ks_out0", - "ks_out1", - "ks_out2", - "lcd0_d2", - "dsi_dp3", - "dsi_dp0", - "sd0_d0", - "sd0_d1", - "sd0_d2_d3", - "sd1_d0_d3", - "sd0_cmd", - "sd0_clk", + "ks_out0_mfp", + "ks_out1_mfp", + "ks_out2_mfp", + "lcd0_d2_mfp", + "dsi_dp3_mfp", + "dsi_dp0_mfp", + "sd0_d0_mfp", + "sd0_d1_mfp", + "sd0_d2_d3_mfp", + "sd1_d0_d3_mfp", + "sd0_cmd_mfp", + "sd0_clk_mfp", }; static const char * const sd1_groups[] = { - "dsi_dp2", - "mfp1_16_14", - "lcd0_d2", - "mfp1_16_14_d17", - "dsi_dp3", - "dsi_dn3", - "dsi_dnp1_cp_d2", - "dsi_dnp1_cp_d17", - "dsi_dn2", - "sd1_d0_d3", - "sd1_cmd", + "dsi_dp2_mfp", + "mfp1_16_14_mfp", + "lcd0_d2_mfp", + "mfp1_16_14_d17_mfp", + "dsi_dp3_mfp", + "dsi_dn3_mfp", + "dsi_dnp1_cp_d2_mfp", + "dsi_dnp1_cp_d17_mfp", + "dsi_dn2_mfp", + "sd1_d0_d3_mfp", + "sd1_cmd_mfp", "sd1_dummy", }; static const char * const sd2_groups[] = { - "dnand_data_wr", + "dnand_data_wr_mfp", }; static const char * const i2c0_groups[] = { - "uart0_rx", - "uart0_tx", - "i2c0_mfp", + "uart0_rx_mfp", + "uart0_tx_mfp", + "i2c0_mfp_mfp", }; static const char * const i2c1_groups[] = { @@ -1448,85 +1448,85 @@ static const char * const i2c2_groups[] = { }; static const char * const i2c3_groups[] = { - "uart2_rx_tx", - "pcm1_sync", - "pcm1_out", + "uart2_rx_tx_mfp", + "pcm1_sync_mfp", + "pcm1_out_mfp", }; static const char * const lvds_groups[] = { - "lvds_o_pn", - "lvds_ee_pn", - "lvds_e_pn", + "lvds_o_pn_mfp", + "lvds_ee_pn_mfp", + "lvds_e_pn_mfp", }; static const char * const bt_groups[] = { - "i2s_pcm1", - "i2s0_pcm0", - "i2s1_pcm0", - "ks_in2", - "ks_in1", - "ks_in0", - "ks_in3", - "ks_out0", - "ks_out1", - "ks_out2", - "lvds_o_pn", - "lvds_ee_pn", - "pcm1_in", - "pcm1_clk", - "pcm1_sync", - "pcm1_out", + "i2s_pcm1_mfp", + "i2s0_pcm0_mfp", + "i2s1_pcm0_mfp", + "ks_in2_mfp", + "ks_in1_mfp", + "ks_in0_mfp", + "ks_in3_mfp", + "ks_out0_mfp", + "ks_out1_mfp", + "ks_out2_mfp", + "lvds_o_pn_mfp", + "lvds_ee_pn_mfp", + "pcm1_in_mfp", + "pcm1_clk_mfp", + "pcm1_sync_mfp", + "pcm1_out_mfp", }; static const char * const lcd0_groups[] = { - "lcd0_d18", - "lcd0_d2", - "mfp1_16_14_d17", - "lvds_o_pn", - "dsi_dp3", - "dsi_dn3", - "lvds_ee_pn", - "dsi_dnp1_cp_d2", - "dsi_dnp1_cp_d17", - "lvds_e_pn", + "lcd0_d18_mfp", + "lcd0_d2_mfp", + "mfp1_16_14_d17_mfp", + "lvds_o_pn_mfp", + "dsi_dp3_mfp", + "dsi_dn3_mfp", + "lvds_ee_pn_mfp", + "dsi_dnp1_cp_d2_mfp", + "dsi_dnp1_cp_d17_mfp", + "lvds_e_pn_mfp", }; static const char * const usb30_groups[] = { - "ks_in1", + "ks_in1_mfp", }; static const char * const clko_25m_groups[] = { - "clko_25m", + "clko_25m_mfp", }; static const char * const mipi_csi_groups[] = { - "csi_cn_cp", - "csi_dn_dp", + "csi_cn_cp_mfp", + "csi_dn_dp_mfp", }; static const char * const dsi_groups[] = { - "dsi_dn0", - "dsi_dp2", - "dsi_dp3", - "dsi_dn3", - "dsi_dp0", - "dsi_dnp1_cp_d2", - "dsi_dnp1_cp_d17", - "dsi_dn2", + "dsi_dn0_mfp", + "dsi_dp2_mfp", + "dsi_dp3_mfp", + "dsi_dn3_mfp", + "dsi_dp0_mfp", + "dsi_dnp1_cp_d2_mfp", + "dsi_dnp1_cp_d17_mfp", + "dsi_dn2_mfp", "dsi_dummy", }; static const char * const nand_groups[] = { - "dnand_data_wr", - "dnand_acle_ce0", - "nand_ceb2", - "nand_ceb3", + "dnand_data_wr_mfp", + "dnand_acle_ce0_mfp", + "nand_ceb2_mfp", + "nand_ceb3_mfp", "nand_dummy", }; static const char * const spdif_groups[] = { - "uart0_tx", + "uart0_tx_mfp", }; static const char * const sirq0_groups[] = { diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c index 0de1a3a96984..06bd2b70af3c 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c @@ -37,12 +37,10 @@ #define MODULE_NAME "pinctrl-bcm2835" #define BCM2835_NUM_GPIOS 54 +#define BCM2711_NUM_GPIOS 58 #define BCM2835_NUM_BANKS 2 #define BCM2835_NUM_IRQS 3 -#define BCM2835_PIN_BITMAP_SZ \ - DIV_ROUND_UP(BCM2835_NUM_GPIOS, sizeof(unsigned long) * 8) - /* GPIO register offsets */ #define GPFSEL0 0x0 /* Function Select */ #define GPSET0 0x1c /* Pin Output Set */ @@ -81,10 +79,11 @@ struct bcm2835_pinctrl { /* note: locking assumes each bank will have its own unsigned long */ unsigned long enabled_irq_map[BCM2835_NUM_BANKS]; - unsigned int irq_type[BCM2835_NUM_GPIOS]; + unsigned int irq_type[BCM2711_NUM_GPIOS]; struct pinctrl_dev *pctl_dev; struct gpio_chip gpio_chip; + struct pinctrl_desc pctl_desc; struct pinctrl_gpio_range gpio_range; raw_spinlock_t irq_lock[BCM2835_NUM_BANKS]; @@ -147,6 +146,10 @@ static struct pinctrl_pin_desc bcm2835_gpio_pins[] = { BCM2835_GPIO_PIN(51), BCM2835_GPIO_PIN(52), BCM2835_GPIO_PIN(53), + BCM2835_GPIO_PIN(54), + BCM2835_GPIO_PIN(55), + BCM2835_GPIO_PIN(56), + BCM2835_GPIO_PIN(57), }; /* one pin per group */ @@ -205,6 +208,10 @@ static const char * const bcm2835_gpio_groups[] = { "gpio51", "gpio52", "gpio53", + "gpio54", + "gpio55", + "gpio56", + "gpio57", }; enum bcm2835_fsel { @@ -322,7 +329,10 @@ static int bcm2835_gpio_get_direction(struct gpio_chip *chip, unsigned int offse if (fsel > BCM2835_FSEL_GPIO_OUT) return -EINVAL; - return (fsel == BCM2835_FSEL_GPIO_IN); + if (fsel == BCM2835_FSEL_GPIO_IN) + return GPIO_LINE_DIRECTION_IN; + + return GPIO_LINE_DIRECTION_OUT; } static void bcm2835_gpio_set(struct gpio_chip *chip, unsigned offset, int value) @@ -355,6 +365,22 @@ static const struct gpio_chip bcm2835_gpio_chip = { .can_sleep = false, }; +static const struct gpio_chip bcm2711_gpio_chip = { + .label = "pinctrl-bcm2711", + .owner = THIS_MODULE, + .request = gpiochip_generic_request, + .free = gpiochip_generic_free, + .direction_input = bcm2835_gpio_direction_input, + .direction_output = bcm2835_gpio_direction_output, + .get_direction = bcm2835_gpio_get_direction, + .get = bcm2835_gpio_get, + .set = bcm2835_gpio_set, + .set_config = gpiochip_generic_config, + .base = -1, + .ngpio = BCM2711_NUM_GPIOS, + .can_sleep = false, +}; + static void bcm2835_gpio_irq_handle_bank(struct bcm2835_pinctrl *pc, unsigned int bank, u32 mask) { @@ -401,7 +427,7 @@ static void bcm2835_gpio_irq_handler(struct irq_desc *desc) bcm2835_gpio_irq_handle_bank(pc, 0, 0xf0000000); bcm2835_gpio_irq_handle_bank(pc, 1, 0x00003fff); break; - case 2: /* IRQ2 covers GPIOs 46-53 */ + case 2: /* IRQ2 covers GPIOs 46-57 */ bcm2835_gpio_irq_handle_bank(pc, 1, 0x003fc000); break; } @@ -620,7 +646,7 @@ static struct irq_chip bcm2835_gpio_irq_chip = { static int bcm2835_pctl_get_groups_count(struct pinctrl_dev *pctldev) { - return ARRAY_SIZE(bcm2835_gpio_groups); + return BCM2835_NUM_GPIOS; } static const char *bcm2835_pctl_get_group_name(struct pinctrl_dev *pctldev, @@ -778,7 +804,7 @@ static int bcm2835_pctl_dt_node_to_map(struct pinctrl_dev *pctldev, err = of_property_read_u32_index(np, "brcm,pins", i, &pin); if (err) goto out; - if (pin >= ARRAY_SIZE(bcm2835_gpio_pins)) { + if (pin >= pc->pctl_desc.npins) { dev_err(pc->dev, "%pOF: invalid brcm,pins value %d\n", np, pin); err = -EINVAL; @@ -854,7 +880,7 @@ static int bcm2835_pmx_get_function_groups(struct pinctrl_dev *pctldev, { /* every pin can do every function */ *groups = bcm2835_gpio_groups; - *num_groups = ARRAY_SIZE(bcm2835_gpio_groups); + *num_groups = BCM2835_NUM_GPIOS; return 0; } @@ -1054,29 +1080,62 @@ static const struct pinconf_ops bcm2711_pinconf_ops = { .pin_config_set = bcm2711_pinconf_set, }; -static struct pinctrl_desc bcm2835_pinctrl_desc = { +static const struct pinctrl_desc bcm2835_pinctrl_desc = { .name = MODULE_NAME, .pins = bcm2835_gpio_pins, - .npins = ARRAY_SIZE(bcm2835_gpio_pins), + .npins = BCM2835_NUM_GPIOS, .pctlops = &bcm2835_pctl_ops, .pmxops = &bcm2835_pmx_ops, .confops = &bcm2835_pinconf_ops, .owner = THIS_MODULE, }; -static struct pinctrl_gpio_range bcm2835_pinctrl_gpio_range = { +static const struct pinctrl_desc bcm2711_pinctrl_desc = { + .name = "pinctrl-bcm2711", + .pins = bcm2835_gpio_pins, + .npins = BCM2711_NUM_GPIOS, + .pctlops = &bcm2835_pctl_ops, + .pmxops = &bcm2835_pmx_ops, + .confops = &bcm2711_pinconf_ops, + .owner = THIS_MODULE, +}; + +static const struct pinctrl_gpio_range bcm2835_pinctrl_gpio_range = { .name = MODULE_NAME, .npins = BCM2835_NUM_GPIOS, }; +static const struct pinctrl_gpio_range bcm2711_pinctrl_gpio_range = { + .name = "pinctrl-bcm2711", + .npins = BCM2711_NUM_GPIOS, +}; + +struct bcm_plat_data { + const struct gpio_chip *gpio_chip; + const struct pinctrl_desc *pctl_desc; + const struct pinctrl_gpio_range *gpio_range; +}; + +static const struct bcm_plat_data bcm2835_plat_data = { + .gpio_chip = &bcm2835_gpio_chip, + .pctl_desc = &bcm2835_pinctrl_desc, + .gpio_range = &bcm2835_pinctrl_gpio_range, +}; + +static const struct bcm_plat_data bcm2711_plat_data = { + .gpio_chip = &bcm2711_gpio_chip, + .pctl_desc = &bcm2711_pinctrl_desc, + .gpio_range = &bcm2711_pinctrl_gpio_range, +}; + static const struct of_device_id bcm2835_pinctrl_match[] = { { .compatible = "brcm,bcm2835-gpio", - .data = &bcm2835_pinconf_ops, + .data = &bcm2835_plat_data, }, { .compatible = "brcm,bcm2711-gpio", - .data = &bcm2711_pinconf_ops, + .data = &bcm2711_plat_data, }, {} }; @@ -1085,14 +1144,15 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; + const struct bcm_plat_data *pdata; struct bcm2835_pinctrl *pc; struct gpio_irq_chip *girq; struct resource iomem; int err, i; const struct of_device_id *match; - BUILD_BUG_ON(ARRAY_SIZE(bcm2835_gpio_pins) != BCM2835_NUM_GPIOS); - BUILD_BUG_ON(ARRAY_SIZE(bcm2835_gpio_groups) != BCM2835_NUM_GPIOS); + BUILD_BUG_ON(ARRAY_SIZE(bcm2835_gpio_pins) != BCM2711_NUM_GPIOS); + BUILD_BUG_ON(ARRAY_SIZE(bcm2835_gpio_groups) != BCM2711_NUM_GPIOS); pc = devm_kzalloc(dev, sizeof(*pc), GFP_KERNEL); if (!pc) @@ -1111,7 +1171,13 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev) if (IS_ERR(pc->base)) return PTR_ERR(pc->base); - pc->gpio_chip = bcm2835_gpio_chip; + match = of_match_node(bcm2835_pinctrl_match, pdev->dev.of_node); + if (!match) + return -EINVAL; + + pdata = match->data; + + pc->gpio_chip = *pdata->gpio_chip; pc->gpio_chip.parent = dev; pc->gpio_chip.of_node = np; @@ -1162,19 +1228,14 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev) return err; } - match = of_match_node(bcm2835_pinctrl_match, pdev->dev.of_node); - if (match) { - bcm2835_pinctrl_desc.confops = - (const struct pinconf_ops *)match->data; - } - - pc->pctl_dev = devm_pinctrl_register(dev, &bcm2835_pinctrl_desc, pc); + pc->pctl_desc = *pdata->pctl_desc; + pc->pctl_dev = devm_pinctrl_register(dev, &pc->pctl_desc, pc); if (IS_ERR(pc->pctl_dev)) { gpiochip_remove(&pc->gpio_chip); return PTR_ERR(pc->pctl_dev); } - pc->gpio_range = bcm2835_pinctrl_gpio_range; + pc->gpio_range = *pdata->gpio_range; pc->gpio_range.base = pc->gpio_chip.base; pc->gpio_range.gc = &pc->gpio_chip; pinctrl_add_gpio_range(pc->pctl_dev, &pc->gpio_range); diff --git a/drivers/pinctrl/bcm/pinctrl-iproc-gpio.c b/drivers/pinctrl/bcm/pinctrl-iproc-gpio.c index 25166217c3e0..a38f0d5f47ce 100644 --- a/drivers/pinctrl/bcm/pinctrl-iproc-gpio.c +++ b/drivers/pinctrl/bcm/pinctrl-iproc-gpio.c @@ -363,7 +363,10 @@ static int iproc_gpio_get_direction(struct gpio_chip *gc, unsigned int gpio) unsigned int offset = IPROC_GPIO_REG(gpio, IPROC_GPIO_OUT_EN_OFFSET); unsigned int shift = IPROC_GPIO_SHIFT(gpio); - return !(readl(chip->base + offset) & BIT(shift)); + if (readl(chip->base + offset) & BIT(shift)) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static void iproc_gpio_set(struct gpio_chip *gc, unsigned gpio, int val) diff --git a/drivers/pinctrl/core.c b/drivers/pinctrl/core.c index f23c55e22195..821242bb4b16 100644 --- a/drivers/pinctrl/core.c +++ b/drivers/pinctrl/core.c @@ -176,6 +176,7 @@ const char *pin_get_name(struct pinctrl_dev *pctldev, const unsigned pin) return desc->name; } +EXPORT_SYMBOL_GPL(pin_get_name); /* Deletes a range of pin descriptors */ static void pinctrl_free_pindescs(struct pinctrl_dev *pctldev, diff --git a/drivers/pinctrl/freescale/Kconfig b/drivers/pinctrl/freescale/Kconfig index de775a85a51e..c784663b00ad 100644 --- a/drivers/pinctrl/freescale/Kconfig +++ b/drivers/pinctrl/freescale/Kconfig @@ -125,28 +125,28 @@ config PINCTRL_IMX7ULP config PINCTRL_IMX8MM bool "IMX8MM pinctrl driver" - depends on ARCH_MXC && ARM64 + depends on ARCH_MXC select PINCTRL_IMX help Say Y here to enable the imx8mm pinctrl driver config PINCTRL_IMX8MN bool "IMX8MN pinctrl driver" - depends on ARCH_MXC && ARM64 + depends on ARCH_MXC select PINCTRL_IMX help Say Y here to enable the imx8mn pinctrl driver config PINCTRL_IMX8MP bool "IMX8MP pinctrl driver" - depends on ARCH_MXC && ARM64 + depends on ARCH_MXC select PINCTRL_IMX help Say Y here to enable the imx8mp pinctrl driver config PINCTRL_IMX8MQ bool "IMX8MQ pinctrl driver" - depends on ARCH_MXC && ARM64 + depends on ARCH_MXC select PINCTRL_IMX help Say Y here to enable the imx8mq pinctrl driver diff --git a/drivers/pinctrl/mediatek/pinctrl-mt6765.c b/drivers/pinctrl/mediatek/pinctrl-mt6765.c index 32451e8693be..905dae8c3fd8 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mt6765.c +++ b/drivers/pinctrl/mediatek/pinctrl-mt6765.c @@ -1070,15 +1070,12 @@ static const struct mtk_pin_soc mt6765_data = { .ngrps = ARRAY_SIZE(mtk_pins_mt6765), .eint_hw = &mt6765_eint_hw, .gpio_m = 0, - .ies_present = true, .base_names = mt6765_pinctrl_register_base_names, .nbase_names = ARRAY_SIZE(mt6765_pinctrl_register_base_names), - .bias_disable_set = mtk_pinconf_bias_disable_set, - .bias_disable_get = mtk_pinconf_bias_disable_get, - .bias_set = mtk_pinconf_bias_set, - .bias_get = mtk_pinconf_bias_get, - .drive_set = mtk_pinconf_drive_set_rev1, - .drive_get = mtk_pinconf_drive_get_rev1, + .bias_set_combo = mtk_pinconf_bias_set_combo, + .bias_get_combo = mtk_pinconf_bias_get_combo, + .drive_set = mtk_pinconf_drive_set_raw, + .drive_get = mtk_pinconf_drive_get_raw, .adv_pull_get = mtk_pinconf_adv_pull_get, .adv_pull_set = mtk_pinconf_adv_pull_set, }; diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8183.c b/drivers/pinctrl/mediatek/pinctrl-mt8183.c index 9a74d5025be6..60318339b618 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mt8183.c +++ b/drivers/pinctrl/mediatek/pinctrl-mt8183.c @@ -554,13 +554,10 @@ static const struct mtk_pin_soc mt8183_data = { .ngrps = ARRAY_SIZE(mtk_pins_mt8183), .eint_hw = &mt8183_eint_hw, .gpio_m = 0, - .ies_present = true, .base_names = mt8183_pinctrl_register_base_names, .nbase_names = ARRAY_SIZE(mt8183_pinctrl_register_base_names), - .bias_disable_set = mtk_pinconf_bias_disable_set_rev1, - .bias_disable_get = mtk_pinconf_bias_disable_get_rev1, - .bias_set = mtk_pinconf_bias_set_rev1, - .bias_get = mtk_pinconf_bias_get_rev1, + .bias_set_combo = mtk_pinconf_bias_set_combo, + .bias_get_combo = mtk_pinconf_bias_get_combo, .drive_set = mtk_pinconf_drive_set_rev1, .drive_get = mtk_pinconf_drive_get_rev1, .adv_pull_get = mtk_pinconf_adv_pull_get, diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c index 20e1c890e73b..d3169a87e1b3 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c @@ -6,6 +6,7 @@ * */ +#include #include #include #include @@ -66,34 +67,44 @@ static int mtk_hw_pin_field_lookup(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, int field, struct mtk_pin_field *pfd) { - const struct mtk_pin_field_calc *c, *e; + const struct mtk_pin_field_calc *c; const struct mtk_pin_reg_calc *rc; + int start = 0, end, check; + bool found = false; u32 bits; if (hw->soc->reg_cal && hw->soc->reg_cal[field].range) { rc = &hw->soc->reg_cal[field]; } else { dev_dbg(hw->dev, - "Not support field %d for pin %d (%s)\n", - field, desc->number, desc->name); + "Not support field %d for this soc\n", field); return -ENOTSUPP; } - c = rc->range; - e = c + rc->nranges; + end = rc->nranges - 1; - while (c < e) { - if (desc->number >= c->s_pin && desc->number <= c->e_pin) + while (start <= end) { + check = (start + end) >> 1; + if (desc->number >= rc->range[check].s_pin + && desc->number <= rc->range[check].e_pin) { + found = true; break; - c++; + } else if (start == end) + break; + else if (desc->number < rc->range[check].s_pin) + end = check - 1; + else + start = check + 1; } - if (c >= e) { + if (!found) { dev_dbg(hw->dev, "Not support field %d for pin = %d (%s)\n", field, desc->number, desc->name); return -ENOTSUPP; } + c = rc->range + check; + if (c->i_base > hw->nbase - 1) { dev_err(hw->dev, "Invalid base for field %d for pin = %d (%s)\n", @@ -182,6 +193,9 @@ int mtk_hw_set_value(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, if (err) return err; + if (value < 0 || value > pf.mask) + return -EINVAL; + if (!pf.next) mtk_rmw(hw, pf.index, pf.offset, pf.mask << pf.bitpos, (value & pf.mask) << pf.bitpos); @@ -502,6 +516,226 @@ int mtk_pinconf_bias_get_rev1(struct mtk_pinctrl *hw, return 0; } +/* Combo for the following pull register type: + * 1. PU + PD + * 2. PULLSEL + PULLEN + * 3. PUPD + R0 + R1 + */ +static int mtk_pinconf_bias_set_pu_pd(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 pullup, u32 arg) +{ + int err, pu, pd; + + if (arg == MTK_DISABLE) { + pu = 0; + pd = 0; + } else if ((arg == MTK_ENABLE) && pullup) { + pu = 1; + pd = 0; + } else if ((arg == MTK_ENABLE) && !pullup) { + pu = 0; + pd = 1; + } else { + err = -EINVAL; + goto out; + } + + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_PU, pu); + if (err) + goto out; + + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_PD, pd); + +out: + return err; +} + +static int mtk_pinconf_bias_set_pullsel_pullen(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 pullup, u32 arg) +{ + int err, enable; + + if (arg == MTK_DISABLE) + enable = 0; + else if (arg == MTK_ENABLE) + enable = 1; + else { + err = -EINVAL; + goto out; + } + + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_PULLEN, enable); + if (err) + goto out; + + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_PULLSEL, pullup); + +out: + return err; +} + +static int mtk_pinconf_bias_set_pupd_r1_r0(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 pullup, u32 arg) +{ + int err, r0, r1; + + if ((arg == MTK_DISABLE) || (arg == MTK_PUPD_SET_R1R0_00)) { + pullup = 0; + r0 = 0; + r1 = 0; + } else if (arg == MTK_PUPD_SET_R1R0_01) { + r0 = 1; + r1 = 0; + } else if (arg == MTK_PUPD_SET_R1R0_10) { + r0 = 0; + r1 = 1; + } else if (arg == MTK_PUPD_SET_R1R0_11) { + r0 = 1; + r1 = 1; + } else { + err = -EINVAL; + goto out; + } + + /* MTK HW PUPD bit: 1 for pull-down, 0 for pull-up */ + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_PUPD, !pullup); + if (err) + goto out; + + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_R0, r0); + if (err) + goto out; + + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_R1, r1); + +out: + return err; +} + +static int mtk_pinconf_bias_get_pu_pd(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 *pullup, u32 *enable) +{ + int err, pu, pd; + + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_PU, &pu); + if (err) + goto out; + + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_PD, &pd); + if (err) + goto out; + + if (pu == 0 && pd == 0) { + *pullup = 0; + *enable = MTK_DISABLE; + } else if (pu == 1 && pd == 0) { + *pullup = 1; + *enable = MTK_ENABLE; + } else if (pu == 0 && pd == 1) { + *pullup = 0; + *enable = MTK_ENABLE; + } else + err = -EINVAL; + +out: + return err; +} + +static int mtk_pinconf_bias_get_pullsel_pullen(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 *pullup, u32 *enable) +{ + int err; + + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_PULLSEL, pullup); + if (err) + goto out; + + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_PULLEN, enable); + +out: + return err; +} + +static int mtk_pinconf_bias_get_pupd_r1_r0(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 *pullup, u32 *enable) +{ + int err, r0, r1; + + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_PUPD, pullup); + if (err) + goto out; + /* MTK HW PUPD bit: 1 for pull-down, 0 for pull-up */ + *pullup = !(*pullup); + + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_R0, &r0); + if (err) + goto out; + + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_R1, &r1); + if (err) + goto out; + + if ((r1 == 0) && (r0 == 0)) + *enable = MTK_PUPD_SET_R1R0_00; + else if ((r1 == 0) && (r0 == 1)) + *enable = MTK_PUPD_SET_R1R0_01; + else if ((r1 == 1) && (r0 == 0)) + *enable = MTK_PUPD_SET_R1R0_10; + else if ((r1 == 1) && (r0 == 1)) + *enable = MTK_PUPD_SET_R1R0_11; + else + err = -EINVAL; + +out: + return err; +} + +int mtk_pinconf_bias_set_combo(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 pullup, u32 arg) +{ + int err; + + err = mtk_pinconf_bias_set_pu_pd(hw, desc, pullup, arg); + if (!err) + goto out; + + err = mtk_pinconf_bias_set_pullsel_pullen(hw, desc, pullup, arg); + if (!err) + goto out; + + err = mtk_pinconf_bias_set_pupd_r1_r0(hw, desc, pullup, arg); + +out: + return err; +} + +int mtk_pinconf_bias_get_combo(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 *pullup, u32 *enable) +{ + int err; + + err = mtk_pinconf_bias_get_pu_pd(hw, desc, pullup, enable); + if (!err) + goto out; + + err = mtk_pinconf_bias_get_pullsel_pullen(hw, desc, pullup, enable); + if (!err) + goto out; + + err = mtk_pinconf_bias_get_pupd_r1_r0(hw, desc, pullup, enable); + +out: + return err; +} + /* Revision 0 */ int mtk_pinconf_drive_set(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, u32 arg) @@ -593,6 +827,18 @@ int mtk_pinconf_drive_get_rev1(struct mtk_pinctrl *hw, return 0; } +int mtk_pinconf_drive_set_raw(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, u32 arg) +{ + return mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DRV, arg); +} + +int mtk_pinconf_drive_get_raw(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, int *val) +{ + return mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DRV, val); +} + int mtk_pinconf_adv_pull_set(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, bool pullup, u32 arg) diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h index 1b7da42aa1d5..27df08736396 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h @@ -216,6 +216,11 @@ struct mtk_pin_soc { int (*bias_get)(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, bool pullup, int *res); + int (*bias_set_combo)(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, u32 pullup, u32 arg); + int (*bias_get_combo)(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, u32 *pullup, u32 *arg); + int (*drive_set)(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, u32 arg); int (*drive_get)(struct mtk_pinctrl *hw, @@ -277,6 +282,12 @@ int mtk_pinconf_bias_set_rev1(struct mtk_pinctrl *hw, int mtk_pinconf_bias_get_rev1(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, bool pullup, int *res); +int mtk_pinconf_bias_set_combo(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 pullup, u32 enable); +int mtk_pinconf_bias_get_combo(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, + u32 *pullup, u32 *enable); int mtk_pinconf_drive_set(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, u32 arg); @@ -288,6 +299,11 @@ int mtk_pinconf_drive_set_rev1(struct mtk_pinctrl *hw, int mtk_pinconf_drive_get_rev1(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, int *val); +int mtk_pinconf_drive_set_raw(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, u32 arg); +int mtk_pinconf_drive_get_raw(struct mtk_pinctrl *hw, + const struct mtk_pin_desc *desc, int *val); + int mtk_pinconf_adv_pull_set(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc, bool pullup, u32 arg); diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c index 67f8444f7a0c..a02ad10ec6fa 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c @@ -804,7 +804,10 @@ static int mtk_gpio_get_direction(struct gpio_chip *chip, unsigned offset) pctl->devdata->spec_dir_set(®_addr, offset); regmap_read(pctl->regmap1, reg_addr, &read_val); - return !(read_val & bit); + if (read_val & bit) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int mtk_gpio_get(struct gpio_chip *chip, unsigned offset) diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c index 923264d0e9ef..3853ec3a2a8e 100644 --- a/drivers/pinctrl/mediatek/pinctrl-paris.c +++ b/drivers/pinctrl/mediatek/pinctrl-paris.c @@ -78,93 +78,88 @@ static int mtk_pinconf_get(struct pinctrl_dev *pctldev, { struct mtk_pinctrl *hw = pinctrl_dev_get_drvdata(pctldev); u32 param = pinconf_to_config_param(*config); - int val, val2, err, reg, ret = 1; + int pullup, err, reg, ret = 1; const struct mtk_pin_desc *desc; + if (pin >= hw->soc->npins) { + err = -EINVAL; + goto out; + } desc = (const struct mtk_pin_desc *)&hw->soc->pins[pin]; switch (param) { case PIN_CONFIG_BIAS_DISABLE: - if (hw->soc->bias_disable_get) { - err = hw->soc->bias_disable_get(hw, desc, &ret); - if (err) - return err; - } else { - return -ENOTSUPP; - } - break; case PIN_CONFIG_BIAS_PULL_UP: - if (hw->soc->bias_get) { - err = hw->soc->bias_get(hw, desc, 1, &ret); - if (err) - return err; - } else { - return -ENOTSUPP; - } - break; case PIN_CONFIG_BIAS_PULL_DOWN: - if (hw->soc->bias_get) { - err = hw->soc->bias_get(hw, desc, 0, &ret); + if (hw->soc->bias_get_combo) { + err = hw->soc->bias_get_combo(hw, desc, &pullup, &ret); if (err) - return err; - } else { - return -ENOTSUPP; - } - break; - case PIN_CONFIG_SLEW_RATE: - err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_SR, &val); - if (err) - return err; - - if (!val) - return -EINVAL; - - break; - case PIN_CONFIG_INPUT_ENABLE: - case PIN_CONFIG_OUTPUT_ENABLE: - err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DIR, &val); - if (err) - return err; - - /* HW takes input mode as zero; output mode as non-zero */ - if ((val && param == PIN_CONFIG_INPUT_ENABLE) || - (!val && param == PIN_CONFIG_OUTPUT_ENABLE)) - return -EINVAL; - - break; - case PIN_CONFIG_INPUT_SCHMITT_ENABLE: - err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DIR, &val); - if (err) - return err; - - err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_SMT, &val2); - if (err) - return err; - - if (val || !val2) - return -EINVAL; - - break; - case PIN_CONFIG_DRIVE_STRENGTH: - if (hw->soc->drive_get) { - err = hw->soc->drive_get(hw, desc, &ret); - if (err) - return err; + goto out; + if (param == PIN_CONFIG_BIAS_DISABLE) { + if (ret == MTK_PUPD_SET_R1R0_00) + ret = MTK_DISABLE; + } else if (param == PIN_CONFIG_BIAS_PULL_UP) { + /* When desire to get pull-up value, return + * error if current setting is pull-down + */ + if (!pullup) + err = -EINVAL; + } else if (param == PIN_CONFIG_BIAS_PULL_DOWN) { + /* When desire to get pull-down value, return + * error if current setting is pull-up + */ + if (pullup) + err = -EINVAL; + } } else { err = -ENOTSUPP; } break; + case PIN_CONFIG_SLEW_RATE: + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_SR, &ret); + break; + case PIN_CONFIG_INPUT_ENABLE: + case PIN_CONFIG_OUTPUT_ENABLE: + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DIR, &ret); + if (err) + goto out; + /* CONFIG Current direction return value + * ------------- ----------------- ---------------------- + * OUTPUT_ENABLE output 1 (= HW value) + * input 0 (= HW value) + * INPUT_ENABLE output 0 (= reverse HW value) + * input 1 (= reverse HW value) + */ + if (param == PIN_CONFIG_INPUT_ENABLE) + ret = !ret; + + break; + case PIN_CONFIG_INPUT_SCHMITT_ENABLE: + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DIR, &ret); + if (err) + goto out; + /* return error when in output mode + * because schmitt trigger only work in input mode + */ + if (ret) { + err = -EINVAL; + goto out; + } + + err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_SMT, &ret); + + break; + case PIN_CONFIG_DRIVE_STRENGTH: + if (hw->soc->drive_get) + err = hw->soc->drive_get(hw, desc, &ret); + else + err = -ENOTSUPP; + break; case MTK_PIN_CONFIG_TDSEL: case MTK_PIN_CONFIG_RDSEL: reg = (param == MTK_PIN_CONFIG_TDSEL) ? PINCTRL_PIN_REG_TDSEL : PINCTRL_PIN_REG_RDSEL; - - err = mtk_hw_get_value(hw, desc, reg, &val); - if (err) - return err; - - ret = val; - + err = mtk_hw_get_value(hw, desc, reg, &ret); break; case MTK_PIN_CONFIG_PU_ADV: case MTK_PIN_CONFIG_PD_ADV: @@ -173,28 +168,24 @@ static int mtk_pinconf_get(struct pinctrl_dev *pctldev, pullup = param == MTK_PIN_CONFIG_PU_ADV; err = hw->soc->adv_pull_get(hw, desc, pullup, &ret); - if (err) - return err; - } else { - return -ENOTSUPP; - } + } else + err = -ENOTSUPP; break; case MTK_PIN_CONFIG_DRV_ADV: - if (hw->soc->adv_drive_get) { + if (hw->soc->adv_drive_get) err = hw->soc->adv_drive_get(hw, desc, &ret); - if (err) - return err; - } else { - return -ENOTSUPP; - } + else + err = -ENOTSUPP; break; default: - return -ENOTSUPP; + err = -ENOTSUPP; } - *config = pinconf_to_config_packed(param, ret); +out: + if (!err) + *config = pinconf_to_config_packed(param, ret); - return 0; + return err; } static int mtk_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin, @@ -206,64 +197,55 @@ static int mtk_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin, int err = 0; u32 reg; + if (pin >= hw->soc->npins) { + err = -EINVAL; + goto err; + } desc = (const struct mtk_pin_desc *)&hw->soc->pins[pin]; switch ((u32)param) { case PIN_CONFIG_BIAS_DISABLE: - if (hw->soc->bias_disable_set) { - err = hw->soc->bias_disable_set(hw, desc); - if (err) - return err; - } else { - return -ENOTSUPP; - } + if (hw->soc->bias_set_combo) + err = hw->soc->bias_set_combo(hw, desc, 0, MTK_DISABLE); + else + err = -ENOTSUPP; break; case PIN_CONFIG_BIAS_PULL_UP: - if (hw->soc->bias_set) { - err = hw->soc->bias_set(hw, desc, 1); - if (err) - return err; - } else { - return -ENOTSUPP; - } + if (hw->soc->bias_set_combo) + err = hw->soc->bias_set_combo(hw, desc, 1, arg); + else + err = -ENOTSUPP; break; case PIN_CONFIG_BIAS_PULL_DOWN: - if (hw->soc->bias_set) { - err = hw->soc->bias_set(hw, desc, 0); - if (err) - return err; - } else { - return -ENOTSUPP; - } + if (hw->soc->bias_set_combo) + err = hw->soc->bias_set_combo(hw, desc, 0, arg); + else + err = -ENOTSUPP; break; case PIN_CONFIG_OUTPUT_ENABLE: err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_SMT, MTK_DISABLE); - if (err) + /* Keep set direction to consider the case that a GPIO pin + * does not have SMT control + */ + if (err != -ENOTSUPP) goto err; err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DIR, MTK_OUTPUT); - if (err) - goto err; break; case PIN_CONFIG_INPUT_ENABLE: - if (hw->soc->ies_present) { - mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_IES, - MTK_ENABLE); - } + /* regard all non-zero value as enable */ + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_IES, !!arg); + if (err) + goto err; err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DIR, MTK_INPUT); - if (err) - goto err; break; case PIN_CONFIG_SLEW_RATE: - err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_SR, - arg); - if (err) - goto err; - + /* regard all non-zero value as enable */ + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_SR, !!arg); break; case PIN_CONFIG_OUTPUT: err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DIR, @@ -273,41 +255,29 @@ static int mtk_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin, err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DO, arg); - if (err) - goto err; break; + case PIN_CONFIG_INPUT_SCHMITT: case PIN_CONFIG_INPUT_SCHMITT_ENABLE: /* arg = 1: Input mode & SMT enable ; * arg = 0: Output mode & SMT disable */ - arg = arg ? 2 : 1; - err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DIR, - arg & 1); + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DIR, !arg); if (err) goto err; - err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_SMT, - !!(arg & 2)); - if (err) - goto err; + err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_SMT, !!arg); break; case PIN_CONFIG_DRIVE_STRENGTH: - if (hw->soc->drive_set) { + if (hw->soc->drive_set) err = hw->soc->drive_set(hw, desc, arg); - if (err) - return err; - } else { - return -ENOTSUPP; - } + else + err = -ENOTSUPP; break; case MTK_PIN_CONFIG_TDSEL: case MTK_PIN_CONFIG_RDSEL: reg = (param == MTK_PIN_CONFIG_TDSEL) ? PINCTRL_PIN_REG_TDSEL : PINCTRL_PIN_REG_RDSEL; - err = mtk_hw_set_value(hw, desc, reg, arg); - if (err) - goto err; break; case MTK_PIN_CONFIG_PU_ADV: case MTK_PIN_CONFIG_PD_ADV: @@ -317,20 +287,14 @@ static int mtk_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin, pullup = param == MTK_PIN_CONFIG_PU_ADV; err = hw->soc->adv_pull_set(hw, desc, pullup, arg); - if (err) - return err; - } else { - return -ENOTSUPP; - } + } else + err = -ENOTSUPP; break; case MTK_PIN_CONFIG_DRV_ADV: - if (hw->soc->adv_drive_set) { + if (hw->soc->adv_drive_set) err = hw->soc->adv_drive_set(hw, desc, arg); - if (err) - return err; - } else { - return -ENOTSUPP; - } + else + err = -ENOTSUPP; break; default: err = -ENOTSUPP; @@ -575,12 +539,120 @@ static int mtk_pctrl_get_group_pins(struct pinctrl_dev *pctldev, return 0; } +static int mtk_hw_get_value_wrap(struct mtk_pinctrl *hw, unsigned int gpio, int field) +{ + const struct mtk_pin_desc *desc; + int value, err; + + if (gpio >= hw->soc->npins) + return -EINVAL; + + desc = (const struct mtk_pin_desc *)&hw->soc->pins[gpio]; + + err = mtk_hw_get_value(hw, desc, field, &value); + if (err) + return err; + + return value; +} + +#define mtk_pctrl_get_pinmux(hw, gpio) \ + mtk_hw_get_value_wrap(hw, gpio, PINCTRL_PIN_REG_MODE) + +#define mtk_pctrl_get_direction(hw, gpio) \ + mtk_hw_get_value_wrap(hw, gpio, PINCTRL_PIN_REG_DIR) + +#define mtk_pctrl_get_out(hw, gpio) \ + mtk_hw_get_value_wrap(hw, gpio, PINCTRL_PIN_REG_DO) + +#define mtk_pctrl_get_in(hw, gpio) \ + mtk_hw_get_value_wrap(hw, gpio, PINCTRL_PIN_REG_DI) + +#define mtk_pctrl_get_smt(hw, gpio) \ + mtk_hw_get_value_wrap(hw, gpio, PINCTRL_PIN_REG_SMT) + +#define mtk_pctrl_get_ies(hw, gpio) \ + mtk_hw_get_value_wrap(hw, gpio, PINCTRL_PIN_REG_IES) + +#define mtk_pctrl_get_driving(hw, gpio) \ + mtk_hw_get_value_wrap(hw, gpio, PINCTRL_PIN_REG_DRV) + +ssize_t mtk_pctrl_show_one_pin(struct mtk_pinctrl *hw, + unsigned int gpio, char *buf, unsigned int bufLen) +{ + int pinmux, pullup, pullen, len = 0, r1 = -1, r0 = -1; + const struct mtk_pin_desc *desc; + + if (gpio >= hw->soc->npins) + return -EINVAL; + + desc = (const struct mtk_pin_desc *)&hw->soc->pins[gpio]; + pinmux = mtk_pctrl_get_pinmux(hw, gpio); + if (pinmux >= hw->soc->nfuncs) + pinmux -= hw->soc->nfuncs; + + mtk_pinconf_bias_get_combo(hw, desc, &pullup, &pullen); + if (pullen == MTK_PUPD_SET_R1R0_00) { + pullen = 0; + r1 = 0; + r0 = 0; + } else if (pullen == MTK_PUPD_SET_R1R0_01) { + pullen = 1; + r1 = 0; + r0 = 1; + } else if (pullen == MTK_PUPD_SET_R1R0_10) { + pullen = 1; + r1 = 1; + r0 = 0; + } else if (pullen == MTK_PUPD_SET_R1R0_11) { + pullen = 1; + r1 = 1; + r0 = 1; + } else if (pullen != MTK_DISABLE && pullen != MTK_ENABLE) { + pullen = 0; + } + len += scnprintf(buf + len, bufLen - len, + "%03d: %1d%1d%1d%1d%02d%1d%1d%1d%1d", + gpio, + pinmux, + mtk_pctrl_get_direction(hw, gpio), + mtk_pctrl_get_out(hw, gpio), + mtk_pctrl_get_in(hw, gpio), + mtk_pctrl_get_driving(hw, gpio), + mtk_pctrl_get_smt(hw, gpio), + mtk_pctrl_get_ies(hw, gpio), + pullen, + pullup); + + if (r1 != -1) { + len += scnprintf(buf + len, bufLen - len, " (%1d %1d)\n", + r1, r0); + } else { + len += scnprintf(buf + len, bufLen - len, "\n"); + } + + return len; +} + +#define PIN_DBG_BUF_SZ 96 +static void mtk_pctrl_dbg_show(struct pinctrl_dev *pctldev, struct seq_file *s, + unsigned int gpio) +{ + struct mtk_pinctrl *hw = pinctrl_dev_get_drvdata(pctldev); + char buf[PIN_DBG_BUF_SZ]; + + (void)mtk_pctrl_show_one_pin(hw, gpio, buf, PIN_DBG_BUF_SZ); + + seq_printf(s, "%s", buf); +} + static const struct pinctrl_ops mtk_pctlops = { .dt_node_to_map = mtk_pctrl_dt_node_to_map, .dt_free_map = pinctrl_utils_free_map, .get_groups_count = mtk_pctrl_get_groups_count, .get_group_name = mtk_pctrl_get_group_name, .get_group_pins = mtk_pctrl_get_group_pins, + .pin_dbg_show = mtk_pctrl_dbg_show, }; static int mtk_pmx_get_funcs_cnt(struct pinctrl_dev *pctldev) @@ -677,6 +749,7 @@ static const struct pinconf_ops mtk_confops = { .pin_config_get = mtk_pinconf_get, .pin_config_group_get = mtk_pconf_group_get, .pin_config_group_set = mtk_pconf_group_set, + .is_generic = true, }; static struct pinctrl_desc mtk_desc = { @@ -693,13 +766,19 @@ static int mtk_gpio_get_direction(struct gpio_chip *chip, unsigned int gpio) const struct mtk_pin_desc *desc; int value, err; + if (gpio >= hw->soc->npins) + return -EINVAL; + desc = (const struct mtk_pin_desc *)&hw->soc->pins[gpio]; err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DIR, &value); if (err) return err; - return !value; + if (value) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int mtk_gpio_get(struct gpio_chip *chip, unsigned int gpio) @@ -708,6 +787,9 @@ static int mtk_gpio_get(struct gpio_chip *chip, unsigned int gpio) const struct mtk_pin_desc *desc; int value, err; + if (gpio >= hw->soc->npins) + return -EINVAL; + desc = (const struct mtk_pin_desc *)&hw->soc->pins[gpio]; err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DI, &value); @@ -722,6 +804,9 @@ static void mtk_gpio_set(struct gpio_chip *chip, unsigned int gpio, int value) struct mtk_pinctrl *hw = gpiochip_get_data(chip); const struct mtk_pin_desc *desc; + if (gpio >= hw->soc->npins) + return; + desc = (const struct mtk_pin_desc *)&hw->soc->pins[gpio]; mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DO, !!value); @@ -729,12 +814,22 @@ static void mtk_gpio_set(struct gpio_chip *chip, unsigned int gpio, int value) static int mtk_gpio_direction_input(struct gpio_chip *chip, unsigned int gpio) { + struct mtk_pinctrl *hw = gpiochip_get_data(chip); + + if (gpio >= hw->soc->npins) + return -EINVAL; + return pinctrl_gpio_direction_input(chip->base + gpio); } static int mtk_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio, int value) { + struct mtk_pinctrl *hw = gpiochip_get_data(chip); + + if (gpio >= hw->soc->npins) + return -EINVAL; + mtk_gpio_set(chip, gpio, value); return pinctrl_gpio_direction_output(chip->base + gpio); diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.h b/drivers/pinctrl/mediatek/pinctrl-paris.h index 3d43771074e6..afb7650fd25b 100644 --- a/drivers/pinctrl/mediatek/pinctrl-paris.h +++ b/drivers/pinctrl/mediatek/pinctrl-paris.h @@ -60,6 +60,9 @@ int mtk_paris_pinctrl_probe(struct platform_device *pdev, const struct mtk_pin_soc *soc); +ssize_t mtk_pctrl_show_one_pin(struct mtk_pinctrl *hw, + unsigned int gpio, char *buf, unsigned int bufLen); + extern const struct dev_pm_ops mtk_paris_pinctrl_pm_ops; #endif /* __PINCTRL_PARIS_H */ diff --git a/drivers/pinctrl/meson/pinctrl-meson-gxbb.c b/drivers/pinctrl/meson/pinctrl-meson-gxbb.c index 926b9997159a..d130c635f74b 100644 --- a/drivers/pinctrl/meson/pinctrl-meson-gxbb.c +++ b/drivers/pinctrl/meson/pinctrl-meson-gxbb.c @@ -231,10 +231,24 @@ static const unsigned int hdmi_hpd_pins[] = { GPIOH_0 }; static const unsigned int hdmi_sda_pins[] = { GPIOH_1 }; static const unsigned int hdmi_scl_pins[] = { GPIOH_2 }; +static const unsigned int tsin_a_d_valid_pins[] = { GPIOY_0 }; +static const unsigned int tsin_a_sop_pins[] = { GPIOY_1 }; +static const unsigned int tsin_a_clk_pins[] = { GPIOY_2 }; +static const unsigned int tsin_a_d0_pins[] = { GPIOY_3 }; +static const unsigned int tsin_a_dp_pins[] = { + GPIOY_4, GPIOY_5, GPIOY_6, GPIOY_7, GPIOY_8, GPIOY_9, GPIOY_10 +}; + +static const unsigned int tsin_a_fail_pins[] = { GPIOY_11 }; static const unsigned int i2s_out_ch23_y_pins[] = { GPIOY_8 }; static const unsigned int i2s_out_ch45_y_pins[] = { GPIOY_9 }; static const unsigned int i2s_out_ch67_y_pins[] = { GPIOY_10 }; +static const unsigned int tsin_b_d_valid_pins[] = { GPIOX_6 }; +static const unsigned int tsin_b_sop_pins[] = { GPIOX_7 }; +static const unsigned int tsin_b_clk_pins[] = { GPIOX_8 }; +static const unsigned int tsin_b_d0_pins[] = { GPIOX_9 }; + static const unsigned int spdif_out_y_pins[] = { GPIOY_12 }; static const unsigned int gen_clk_out_pins[] = { GPIOY_15 }; @@ -437,12 +451,22 @@ static struct meson_pmx_group meson_gxbb_periphs_groups[] = { GROUP(pwm_a_x, 3, 17), GROUP(pwm_e, 2, 30), GROUP(pwm_f_x, 3, 18), + GROUP(tsin_b_d_valid, 3, 9), + GROUP(tsin_b_sop, 3, 8), + GROUP(tsin_b_clk, 3, 10), + GROUP(tsin_b_d0, 3, 7), /* Bank Y */ GROUP(uart_cts_c, 1, 17), GROUP(uart_rts_c, 1, 16), GROUP(uart_tx_c, 1, 19), GROUP(uart_rx_c, 1, 18), + GROUP(tsin_a_fail, 3, 3), + GROUP(tsin_a_d_valid, 3, 2), + GROUP(tsin_a_sop, 3, 1), + GROUP(tsin_a_clk, 3, 0), + GROUP(tsin_a_d0, 3, 4), + GROUP(tsin_a_dp, 3, 5), GROUP(pwm_a_y, 1, 21), GROUP(pwm_f_y, 1, 20), GROUP(i2s_out_ch23_y, 1, 5), @@ -601,6 +625,15 @@ static const char * const gpio_periphs_groups[] = { "GPIOX_20", "GPIOX_21", "GPIOX_22", }; +static const char * const tsin_a_groups[] = { + "tsin_a_clk", "tsin_a_sop", "tsin_a_d_valid", "tsin_a_d0", + "tsin_a_dp", "tsin_a_fail", +}; + +static const char * const tsin_b_groups[] = { + "tsin_b_clk", "tsin_b_sop", "tsin_b_d_valid", "tsin_b_d0", +}; + static const char * const emmc_groups[] = { "emmc_nand_d07", "emmc_clk", "emmc_cmd", "emmc_ds", }; @@ -792,6 +825,8 @@ static struct meson_pmx_func meson_gxbb_periphs_functions[] = { FUNCTION(i2s_out), FUNCTION(spdif_out), FUNCTION(gen_clk_out), + FUNCTION(tsin_a), + FUNCTION(tsin_b), }; static struct meson_pmx_func meson_gxbb_aobus_functions[] = { diff --git a/drivers/pinctrl/meson/pinctrl-meson-gxl.c b/drivers/pinctrl/meson/pinctrl-meson-gxl.c index 2ac921c83da9..32552d795bb2 100644 --- a/drivers/pinctrl/meson/pinctrl-meson-gxl.c +++ b/drivers/pinctrl/meson/pinctrl-meson-gxl.c @@ -241,6 +241,17 @@ static const unsigned int tsin_a_dp_pins[] = { GPIODV_1, GPIODV_2, GPIODV_3, GPIODV_4, GPIODV_5, GPIODV_6, GPIODV_7, }; +static const unsigned int tsin_b_clk_pins[] = { GPIOH_6 }; +static const unsigned int tsin_b_d0_pins[] = { GPIOH_7 }; +static const unsigned int tsin_b_sop_pins[] = { GPIOH_8 }; +static const unsigned int tsin_b_d_valid_pins[] = { GPIOH_9 }; + +static const unsigned int tsin_b_fail_z4_pins[] = { GPIOZ_4 }; +static const unsigned int tsin_b_clk_z3_pins[] = { GPIOZ_3 }; +static const unsigned int tsin_b_d0_z2_pins[] = { GPIOZ_2 }; +static const unsigned int tsin_b_sop_z1_pins[] = { GPIOZ_1 }; +static const unsigned int tsin_b_d_valid_z0_pins[] = { GPIOZ_0 }; + static const struct pinctrl_pin_desc meson_gxl_aobus_pins[] = { MESON_PIN(GPIOAO_0), MESON_PIN(GPIOAO_1), @@ -438,6 +449,11 @@ static struct meson_pmx_group meson_gxl_periphs_groups[] = { GROUP(eth_txd1, 4, 12), GROUP(eth_txd2, 4, 11), GROUP(eth_txd3, 4, 10), + GROUP(tsin_b_fail_z4, 3, 15), + GROUP(tsin_b_clk_z3, 3, 16), + GROUP(tsin_b_d0_z2, 3, 17), + GROUP(tsin_b_sop_z1, 3, 18), + GROUP(tsin_b_d_valid_z0, 3, 19), GROUP(pwm_c, 3, 20), GROUP(i2s_out_ch23_z, 3, 26), GROUP(i2s_out_ch45_z, 3, 25), @@ -454,6 +470,10 @@ static struct meson_pmx_group meson_gxl_periphs_groups[] = { GROUP(i2s_out_lr_clk, 6, 24), GROUP(i2s_out_ch01, 6, 23), GROUP(spdif_out_h, 6, 28), + GROUP(tsin_b_d0, 6, 17), + GROUP(tsin_b_sop, 6, 18), + GROUP(tsin_b_d_valid, 6, 19), + GROUP(tsin_b_clk, 6, 20), /* Bank DV */ GROUP(uart_tx_b, 2, 16), @@ -689,6 +709,12 @@ static const char * const tsin_a_groups[] = { "tsin_a_dp", "tsin_a_fail", }; +static const char * const tsin_b_groups[] = { + "tsin_b_clk", "tsin_b_sop", "tsin_b_d_valid", "tsin_b_d0", + "tsin_b_clk_z3", "tsin_b_sop_z1", "tsin_b_d_valid_z0", "tsin_b_d0_z2", + "tsin_b_fail_z4", +}; + static const char * const gpio_aobus_groups[] = { "GPIOAO_0", "GPIOAO_1", "GPIOAO_2", "GPIOAO_3", "GPIOAO_4", "GPIOAO_5", "GPIOAO_6", "GPIOAO_7", "GPIOAO_8", "GPIOAO_9", @@ -764,6 +790,7 @@ static struct meson_pmx_func meson_gxl_periphs_functions[] = { FUNCTION(spdif_out), FUNCTION(eth_led), FUNCTION(tsin_a), + FUNCTION(tsin_b), }; static struct meson_pmx_func meson_gxl_aobus_functions[] = { diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c index 243fba254175..5f125bd6279d 100644 --- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c +++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -402,7 +403,10 @@ static int armada_37xx_gpio_get_direction(struct gpio_chip *chip, mask = BIT(offset); regmap_read(info->regmap, reg, &val); - return !(val & mask); + if (val & mask) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int armada_37xx_gpio_direction_output(struct gpio_chip *chip, @@ -738,14 +742,7 @@ static int armada_37xx_irqchip_register(struct platform_device *pdev, return ret; } - nr_irq_parent = platform_irq_count(pdev); - if (nr_irq_parent < 0) { - if (nr_irq_parent != -EPROBE_DEFER) - dev_err(dev, "Couldn't determine irq count: %pe\n", - ERR_PTR(nr_irq_parent)); - return nr_irq_parent; - } - + nr_irq_parent = of_irq_count(np); spin_lock_init(&info->irq_lock); if (!nr_irq_parent) { @@ -782,7 +779,7 @@ static int armada_37xx_irqchip_register(struct platform_device *pdev, if (!girq->parents) return -ENOMEM; for (i = 0; i < nr_irq_parent; i++) { - int irq = platform_get_irq(pdev, i); + int irq = irq_of_parse_and_map(np, i); if (irq < 0) continue; diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c index 95f864dfdef4..ca7bbe4164c0 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c @@ -831,11 +831,14 @@ static int nmk_gpio_get_dir(struct gpio_chip *chip, unsigned offset) clk_enable(nmk_chip->clk); - dir = !(readl(nmk_chip->addr + NMK_GPIO_DIR) & BIT(offset)); + dir = readl(nmk_chip->addr + NMK_GPIO_DIR) & BIT(offset); clk_disable(nmk_chip->clk); - return dir; + if (dir) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int nmk_gpio_make_input(struct gpio_chip *chip, unsigned offset) diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c index 9eb86309c70b..dfef471201f6 100644 --- a/drivers/pinctrl/pinconf-generic.c +++ b/drivers/pinctrl/pinconf-generic.c @@ -286,6 +286,7 @@ int pinconf_generic_parse_dt_config(struct device_node *np, kfree(cfg); return ret; } +EXPORT_SYMBOL_GPL(pinconf_generic_parse_dt_config); int pinconf_generic_dt_subnode_to_map(struct pinctrl_dev *pctldev, struct device_node *np, struct pinctrl_map **map, diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c index 73aff6591de2..1fe62a35bb12 100644 --- a/drivers/pinctrl/pinctrl-amd.c +++ b/drivers/pinctrl/pinctrl-amd.c @@ -46,7 +46,10 @@ static int amd_gpio_get_direction(struct gpio_chip *gc, unsigned offset) pin_reg = readl(gpio_dev->base + offset * 4); raw_spin_unlock_irqrestore(&gpio_dev->lock, flags); - return !(pin_reg & BIT(OUTPUT_ENABLE_OFF)); + if (pin_reg & BIT(OUTPUT_ENABLE_OFF)) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int amd_gpio_direction_input(struct gpio_chip *gc, unsigned offset) diff --git a/drivers/pinctrl/pinctrl-at91.c b/drivers/pinctrl/pinctrl-at91.c index 207f266e9cf2..52386ad29f28 100644 --- a/drivers/pinctrl/pinctrl-at91.c +++ b/drivers/pinctrl/pinctrl-at91.c @@ -1414,7 +1414,10 @@ static int at91_gpio_get_direction(struct gpio_chip *chip, unsigned offset) u32 osr; osr = readl_relaxed(pio + PIO_OSR); - return !(osr & mask); + if (osr & mask) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int at91_gpio_direction_input(struct gpio_chip *chip, unsigned offset) diff --git a/drivers/pinctrl/pinctrl-axp209.c b/drivers/pinctrl/pinctrl-axp209.c index be5b645815e5..207cbae3a7bf 100644 --- a/drivers/pinctrl/pinctrl-axp209.c +++ b/drivers/pinctrl/pinctrl-axp209.c @@ -149,13 +149,16 @@ static int axp20x_gpio_get_direction(struct gpio_chip *chip, * going to change the value soon anyway. Default to output. */ if ((val & AXP20X_GPIO_FUNCTIONS) > 2) - return 0; + return GPIO_LINE_DIRECTION_OUT; /* * The GPIO directions are the three lowest values. * 2 is input, 0 and 1 are output */ - return val & 2; + if (val & 2) + return GPIO_LINE_DIRECTION_IN; + + return GPIO_LINE_DIRECTION_OUT; } static int axp20x_gpio_output(struct gpio_chip *chip, unsigned int offset, diff --git a/drivers/pinctrl/pinctrl-da9062.c b/drivers/pinctrl/pinctrl-da9062.c new file mode 100644 index 000000000000..1c08579f0198 --- /dev/null +++ b/drivers/pinctrl/pinctrl-da9062.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Dialog DA9062 pinctrl and GPIO driver. + * Based on DA9055 GPIO driver. + * + * TODO: + * - add pinmux and pinctrl support (gpio alternate mode) + * + * Documents: + * [1] https://www.dialog-semiconductor.com/sites/default/files/da9062_datasheet_3v6.pdf + * + * Copyright (C) 2019 Pengutronix, Marco Felsch + */ +#include +#include +#include +#include + +#include + +#include +#include + +/* + * We need this get the gpio_desc from a tuple to decide if + * the gpio is active low without a vendor specific dt-binding. + */ +#include "../gpio/gpiolib.h" + +#define DA9062_TYPE(offset) (4 * (offset % 2)) +#define DA9062_PIN_SHIFT(offset) (4 * (offset % 2)) +#define DA9062_PIN_ALTERNATE 0x00 /* gpio alternate mode */ +#define DA9062_PIN_GPI 0x01 /* gpio in */ +#define DA9062_PIN_GPO_OD 0x02 /* gpio out open-drain */ +#define DA9062_PIN_GPO_PP 0x03 /* gpio out push-pull */ +#define DA9062_GPIO_NUM 5 + +struct da9062_pctl { + struct da9062 *da9062; + struct gpio_chip gc; + unsigned int pin_config[DA9062_GPIO_NUM]; +}; + +static int da9062_pctl_get_pin_mode(struct da9062_pctl *pctl, + unsigned int offset) +{ + struct regmap *regmap = pctl->da9062->regmap; + int ret, val; + + ret = regmap_read(regmap, DA9062AA_GPIO_0_1 + (offset >> 1), &val); + if (ret < 0) + return ret; + + val >>= DA9062_PIN_SHIFT(offset); + val &= DA9062AA_GPIO0_PIN_MASK; + + return val; +} + +static int da9062_pctl_set_pin_mode(struct da9062_pctl *pctl, + unsigned int offset, unsigned int mode_req) +{ + struct regmap *regmap = pctl->da9062->regmap; + unsigned int mode = mode_req; + unsigned int mask; + int ret; + + mode &= DA9062AA_GPIO0_PIN_MASK; + mode <<= DA9062_PIN_SHIFT(offset); + mask = DA9062AA_GPIO0_PIN_MASK << DA9062_PIN_SHIFT(offset); + + ret = regmap_update_bits(regmap, DA9062AA_GPIO_0_1 + (offset >> 1), + mask, mode); + if (!ret) + pctl->pin_config[offset] = mode_req; + + return ret; +} + +static int da9062_gpio_get(struct gpio_chip *gc, unsigned int offset) +{ + struct da9062_pctl *pctl = gpiochip_get_data(gc); + struct regmap *regmap = pctl->da9062->regmap; + int gpio_mode, val; + int ret; + + gpio_mode = da9062_pctl_get_pin_mode(pctl, offset); + if (gpio_mode < 0) + return gpio_mode; + + switch (gpio_mode) { + case DA9062_PIN_ALTERNATE: + return -ENOTSUPP; + case DA9062_PIN_GPI: + ret = regmap_read(regmap, DA9062AA_STATUS_B, &val); + if (ret < 0) + return ret; + break; + case DA9062_PIN_GPO_OD: + case DA9062_PIN_GPO_PP: + ret = regmap_read(regmap, DA9062AA_GPIO_MODE0_4, &val); + if (ret < 0) + return ret; + } + + return !!(val & BIT(offset)); +} + +static void da9062_gpio_set(struct gpio_chip *gc, unsigned int offset, + int value) +{ + struct da9062_pctl *pctl = gpiochip_get_data(gc); + struct regmap *regmap = pctl->da9062->regmap; + + regmap_update_bits(regmap, DA9062AA_GPIO_MODE0_4, BIT(offset), + value << offset); +} + +static int da9062_gpio_get_direction(struct gpio_chip *gc, unsigned int offset) +{ + struct da9062_pctl *pctl = gpiochip_get_data(gc); + int gpio_mode; + + gpio_mode = da9062_pctl_get_pin_mode(pctl, offset); + if (gpio_mode < 0) + return gpio_mode; + + switch (gpio_mode) { + case DA9062_PIN_ALTERNATE: + return -ENOTSUPP; + case DA9062_PIN_GPI: + return GPIO_LINE_DIRECTION_IN; + case DA9062_PIN_GPO_OD: + case DA9062_PIN_GPO_PP: + return GPIO_LINE_DIRECTION_OUT; + } + + return -EINVAL; +} + +static int da9062_gpio_direction_input(struct gpio_chip *gc, + unsigned int offset) +{ + struct da9062_pctl *pctl = gpiochip_get_data(gc); + struct regmap *regmap = pctl->da9062->regmap; + struct gpio_desc *desc = gpiochip_get_desc(gc, offset); + unsigned int gpi_type; + int ret; + + ret = da9062_pctl_set_pin_mode(pctl, offset, DA9062_PIN_GPI); + if (ret) + return ret; + + /* + * If the gpio is active low we should set it in hw too. No worries + * about gpio_get() because we read and return the gpio-level. So the + * gpiolib active_low handling is still correct. + * + * 0 - active low, 1 - active high + */ + gpi_type = !gpiod_is_active_low(desc); + + return regmap_update_bits(regmap, DA9062AA_GPIO_0_1 + (offset >> 1), + DA9062AA_GPIO0_TYPE_MASK << DA9062_TYPE(offset), + gpi_type << DA9062_TYPE(offset)); +} + +static int da9062_gpio_direction_output(struct gpio_chip *gc, + unsigned int offset, int value) +{ + struct da9062_pctl *pctl = gpiochip_get_data(gc); + unsigned int pin_config = pctl->pin_config[offset]; + int ret; + + ret = da9062_pctl_set_pin_mode(pctl, offset, pin_config); + if (ret) + return ret; + + da9062_gpio_set(gc, offset, value); + + return 0; +} + +static int da9062_gpio_set_config(struct gpio_chip *gc, unsigned int offset, + unsigned long config) +{ + struct da9062_pctl *pctl = gpiochip_get_data(gc); + struct regmap *regmap = pctl->da9062->regmap; + int gpio_mode; + + /* + * We need to meet the following restrictions [1, Figure 18]: + * - PIN_CONFIG_BIAS_PULL_DOWN -> only allowed if the pin is used as + * gpio input + * - PIN_CONFIG_BIAS_PULL_UP -> only allowed if the pin is used as + * gpio output open-drain. + */ + + switch (pinconf_to_config_param(config)) { + case PIN_CONFIG_BIAS_DISABLE: + return regmap_update_bits(regmap, DA9062AA_CONFIG_K, + BIT(offset), 0); + case PIN_CONFIG_BIAS_PULL_DOWN: + gpio_mode = da9062_pctl_get_pin_mode(pctl, offset); + if (gpio_mode < 0) + return -EINVAL; + else if (gpio_mode != DA9062_PIN_GPI) + return -ENOTSUPP; + return regmap_update_bits(regmap, DA9062AA_CONFIG_K, + BIT(offset), BIT(offset)); + case PIN_CONFIG_BIAS_PULL_UP: + gpio_mode = da9062_pctl_get_pin_mode(pctl, offset); + if (gpio_mode < 0) + return -EINVAL; + else if (gpio_mode != DA9062_PIN_GPO_OD) + return -ENOTSUPP; + return regmap_update_bits(regmap, DA9062AA_CONFIG_K, + BIT(offset), BIT(offset)); + case PIN_CONFIG_DRIVE_OPEN_DRAIN: + return da9062_pctl_set_pin_mode(pctl, offset, + DA9062_PIN_GPO_OD); + case PIN_CONFIG_DRIVE_PUSH_PULL: + return da9062_pctl_set_pin_mode(pctl, offset, + DA9062_PIN_GPO_PP); + default: + return -ENOTSUPP; + } +} + +static int da9062_gpio_to_irq(struct gpio_chip *gc, unsigned int offset) +{ + struct da9062_pctl *pctl = gpiochip_get_data(gc); + struct da9062 *da9062 = pctl->da9062; + + return regmap_irq_get_virq(da9062->regmap_irq, + DA9062_IRQ_GPI0 + offset); +} + +static const struct gpio_chip reference_gc = { + .owner = THIS_MODULE, + .get = da9062_gpio_get, + .set = da9062_gpio_set, + .get_direction = da9062_gpio_get_direction, + .direction_input = da9062_gpio_direction_input, + .direction_output = da9062_gpio_direction_output, + .set_config = da9062_gpio_set_config, + .to_irq = da9062_gpio_to_irq, + .can_sleep = true, + .ngpio = DA9062_GPIO_NUM, + .base = -1, +}; + +static int da9062_pctl_probe(struct platform_device *pdev) +{ + struct device *parent = pdev->dev.parent; + struct da9062_pctl *pctl; + int i; + + pctl = devm_kzalloc(&pdev->dev, sizeof(*pctl), GFP_KERNEL); + if (!pctl) + return -ENOMEM; + + pctl->da9062 = dev_get_drvdata(parent); + if (!pctl->da9062) + return -EINVAL; + + if (!device_property_present(parent, "gpio-controller")) + return 0; + + for (i = 0; i < ARRAY_SIZE(pctl->pin_config); i++) + pctl->pin_config[i] = DA9062_PIN_GPO_PP; + + /* + * Currently the driver handles only the GPIO support. The + * pinctrl/pinmux support can be added later if needed. + */ + pctl->gc = reference_gc; + pctl->gc.label = dev_name(&pdev->dev); + pctl->gc.parent = &pdev->dev; +#ifdef CONFIG_OF_GPIO + pctl->gc.of_node = parent->of_node; +#endif + + platform_set_drvdata(pdev, pctl); + + return devm_gpiochip_add_data(&pdev->dev, &pctl->gc, pctl); +} + +static struct platform_driver da9062_pctl_driver = { + .probe = da9062_pctl_probe, + .driver = { + .name = "da9062-gpio", + }, +}; +module_platform_driver(da9062_pctl_driver); + +MODULE_AUTHOR("Marco Felsch "); +MODULE_DESCRIPTION("DA9062 PMIC pinctrl and GPIO Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:da9062-gpio"); diff --git a/drivers/pinctrl/pinctrl-ingenic.c b/drivers/pinctrl/pinctrl-ingenic.c index 96f04d121ebd..7f5376ce7fb1 100644 --- a/drivers/pinctrl/pinctrl-ingenic.c +++ b/drivers/pinctrl/pinctrl-ingenic.c @@ -4,6 +4,7 @@ * * Copyright (c) 2017 Paul Cercueil * Copyright (c) 2019 周琰杰 (Zhou Yanjie) + * Copyright (c) 2017, 2019 Paul Boddie */ #include @@ -900,6 +901,7 @@ static int jz4780_mmc0_8bit_a_pins[] = { 0x04, 0x05, 0x06, 0x07, 0x18, }; static int jz4780_i2c3_pins[] = { 0x6a, 0x6b, }; static int jz4780_i2c4_e_pins[] = { 0x8c, 0x8d, }; static int jz4780_i2c4_f_pins[] = { 0xb9, 0xb8, }; +static int jz4780_hdmi_ddc_pins[] = { 0xb9, 0xb8, }; static int jz4780_uart2_data_funcs[] = { 1, 1, }; static int jz4780_uart2_hwflow_funcs[] = { 1, 1, }; @@ -908,6 +910,7 @@ static int jz4780_mmc0_8bit_a_funcs[] = { 1, 1, 1, 1, 1, }; static int jz4780_i2c3_funcs[] = { 1, 1, }; static int jz4780_i2c4_e_funcs[] = { 1, 1, }; static int jz4780_i2c4_f_funcs[] = { 1, 1, }; +static int jz4780_hdmi_ddc_funcs[] = { 0, 0, }; static const struct group_desc jz4780_groups[] = { INGENIC_PIN_GROUP("uart0-data", jz4770_uart0_data), @@ -950,6 +953,7 @@ static const struct group_desc jz4780_groups[] = { INGENIC_PIN_GROUP("i2c3-data", jz4780_i2c3), INGENIC_PIN_GROUP("i2c4-data-e", jz4780_i2c4_e), INGENIC_PIN_GROUP("i2c4-data-f", jz4780_i2c4_f), + INGENIC_PIN_GROUP("hdmi-ddc", jz4780_hdmi_ddc), INGENIC_PIN_GROUP("cim-data", jz4770_cim_8bit), INGENIC_PIN_GROUP("lcd-24bit", jz4770_lcd_24bit), { "lcd-no-pins", }, @@ -982,6 +986,7 @@ static const char *jz4780_nemc_groups[] = { static const char *jz4780_i2c3_groups[] = { "i2c3-data", }; static const char *jz4780_i2c4_groups[] = { "i2c4-data-e", "i2c4-data-f", }; static const char *jz4780_cim_groups[] = { "cim-data", }; +static const char *jz4780_hdmi_ddc_groups[] = { "hdmi-ddc", }; static const struct function_desc jz4780_functions[] = { { "uart0", jz4770_uart0_groups, ARRAY_SIZE(jz4770_uart0_groups), }, @@ -1014,6 +1019,8 @@ static const struct function_desc jz4780_functions[] = { { "pwm5", jz4770_pwm5_groups, ARRAY_SIZE(jz4770_pwm5_groups), }, { "pwm6", jz4770_pwm6_groups, ARRAY_SIZE(jz4770_pwm6_groups), }, { "pwm7", jz4770_pwm7_groups, ARRAY_SIZE(jz4770_pwm7_groups), }, + { "hdmi-ddc", jz4780_hdmi_ddc_groups, + ARRAY_SIZE(jz4780_hdmi_ddc_groups), }, }; static const struct ingenic_chip_info jz4780_chip_info = { @@ -1437,6 +1444,19 @@ static int x1830_mmc1_4bit_pins[] = { 0x45, 0x46, 0x47, }; static int x1830_i2c0_pins[] = { 0x0c, 0x0d, }; static int x1830_i2c1_pins[] = { 0x39, 0x3a, }; static int x1830_i2c2_pins[] = { 0x5b, 0x5c, }; +static int x1830_lcd_rgb_18bit_pins[] = { + 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, +}; +static int x1830_lcd_slcd_8bit_pins[] = { + 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x6c, 0x6d, + 0x69, 0x72, 0x73, 0x7b, 0x7a, +}; +static int x1830_lcd_slcd_16bit_pins[] = { + 0x6e, 0x6f, 0x70, 0x71, 0x76, 0x77, 0x78, 0x79, +}; static int x1830_pwm_pwm0_b_pins[] = { 0x31, }; static int x1830_pwm_pwm0_c_pins[] = { 0x4b, }; static int x1830_pwm_pwm1_b_pins[] = { 0x32, }; @@ -1486,6 +1506,16 @@ static int x1830_mmc1_4bit_funcs[] = { 0, 0, 0, }; static int x1830_i2c0_funcs[] = { 1, 1, }; static int x1830_i2c1_funcs[] = { 0, 0, }; static int x1830_i2c2_funcs[] = { 1, 1, }; +static int x1830_lcd_rgb_18bit_funcs[] = { + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, +}; +static int x1830_lcd_slcd_8bit_funcs[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +}; +static int x1830_lcd_slcd_16bit_funcs[] = { 1, 1, 1, 1, 1, 1, 1, 1, }; static int x1830_pwm_pwm0_b_funcs[] = { 0, }; static int x1830_pwm_pwm0_c_funcs[] = { 1, }; static int x1830_pwm_pwm1_b_funcs[] = { 0, }; @@ -1534,6 +1564,10 @@ static const struct group_desc x1830_groups[] = { INGENIC_PIN_GROUP("i2c0-data", x1830_i2c0), INGENIC_PIN_GROUP("i2c1-data", x1830_i2c1), INGENIC_PIN_GROUP("i2c2-data", x1830_i2c2), + INGENIC_PIN_GROUP("lcd-rgb-18bit", x1830_lcd_rgb_18bit), + INGENIC_PIN_GROUP("lcd-slcd-8bit", x1830_lcd_slcd_8bit), + INGENIC_PIN_GROUP("lcd-slcd-16bit", x1830_lcd_slcd_16bit), + { "lcd-no-pins", }, INGENIC_PIN_GROUP("pwm0-b", x1830_pwm_pwm0_b), INGENIC_PIN_GROUP("pwm0-c", x1830_pwm_pwm0_c), INGENIC_PIN_GROUP("pwm1-b", x1830_pwm_pwm1_b), @@ -1572,6 +1606,9 @@ static const char *x1830_mmc1_groups[] = { "mmc1-1bit", "mmc1-4bit", }; static const char *x1830_i2c0_groups[] = { "i2c0-data", }; static const char *x1830_i2c1_groups[] = { "i2c1-data", }; static const char *x1830_i2c2_groups[] = { "i2c2-data", }; +static const char *x1830_lcd_groups[] = { + "lcd-rgb-18bit", "lcd-slcd-8bit", "lcd-slcd-16bit", "lcd-no-pins", +}; static const char *x1830_pwm0_groups[] = { "pwm0-b", "pwm0-c", }; static const char *x1830_pwm1_groups[] = { "pwm1-b", "pwm1-c", }; static const char *x1830_pwm2_groups[] = { "pwm2-c-8", "pwm2-c-13", }; @@ -1593,6 +1630,7 @@ static const struct function_desc x1830_functions[] = { { "i2c0", x1830_i2c0_groups, ARRAY_SIZE(x1830_i2c0_groups), }, { "i2c1", x1830_i2c1_groups, ARRAY_SIZE(x1830_i2c1_groups), }, { "i2c2", x1830_i2c2_groups, ARRAY_SIZE(x1830_i2c2_groups), }, + { "lcd", x1830_lcd_groups, ARRAY_SIZE(x1830_lcd_groups), }, { "pwm0", x1830_pwm0_groups, ARRAY_SIZE(x1830_pwm0_groups), }, { "pwm1", x1830_pwm1_groups, ARRAY_SIZE(x1830_pwm1_groups), }, { "pwm2", x1830_pwm2_groups, ARRAY_SIZE(x1830_pwm2_groups), }, @@ -1916,13 +1954,19 @@ static int ingenic_gpio_get_direction(struct gpio_chip *gc, unsigned int offset) struct ingenic_pinctrl *jzpc = jzgc->jzpc; unsigned int pin = gc->base + offset; - if (jzpc->info->version >= ID_JZ4760) - return ingenic_get_pin_config(jzpc, pin, JZ4760_GPIO_PAT1); + if (jzpc->info->version >= ID_JZ4760) { + if (ingenic_get_pin_config(jzpc, pin, JZ4760_GPIO_PAT1)) + return GPIO_LINE_DIRECTION_IN; + return GPIO_LINE_DIRECTION_OUT; + } if (ingenic_get_pin_config(jzpc, pin, JZ4740_GPIO_SELECT)) - return true; + return GPIO_LINE_DIRECTION_IN; - return !ingenic_get_pin_config(jzpc, pin, JZ4740_GPIO_DIR); + if (ingenic_get_pin_config(jzpc, pin, JZ4740_GPIO_DIR)) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static const struct pinctrl_ops ingenic_pctlops = { @@ -2158,7 +2202,8 @@ static int ingenic_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin, break; default: - unreachable(); + /* unreachable */ + break; } } diff --git a/drivers/pinctrl/pinctrl-ocelot.c b/drivers/pinctrl/pinctrl-ocelot.c index eb3dd0d46d6c..ed8eac6c1494 100644 --- a/drivers/pinctrl/pinctrl-ocelot.c +++ b/drivers/pinctrl/pinctrl-ocelot.c @@ -604,7 +604,10 @@ static int ocelot_gpio_get_direction(struct gpio_chip *chip, regmap_read(info->map, REG(OCELOT_GPIO_OE, info, offset), &val); - return !(val & BIT(offset % 32)); + if (val & BIT(offset % 32)) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int ocelot_gpio_direction_input(struct gpio_chip *chip, diff --git a/drivers/pinctrl/pinctrl-oxnas.c b/drivers/pinctrl/pinctrl-oxnas.c index 674b7b5919df..5a312279b3c7 100644 --- a/drivers/pinctrl/pinctrl-oxnas.c +++ b/drivers/pinctrl/pinctrl-oxnas.c @@ -756,7 +756,10 @@ static int oxnas_gpio_get_direction(struct gpio_chip *chip, struct oxnas_gpio_bank *bank = gpiochip_get_data(chip); u32 mask = BIT(offset); - return !(readl_relaxed(bank->reg_base + OUTPUT_EN) & mask); + if (readl_relaxed(bank->reg_base + OUTPUT_EN) & mask) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int oxnas_gpio_direction_input(struct gpio_chip *chip, diff --git a/drivers/pinctrl/pinctrl-pic32.c b/drivers/pinctrl/pinctrl-pic32.c index e5d6d3f9753e..a6e2a4a4ca95 100644 --- a/drivers/pinctrl/pinctrl-pic32.c +++ b/drivers/pinctrl/pinctrl-pic32.c @@ -1990,7 +1990,10 @@ static int pic32_gpio_get_direction(struct gpio_chip *chip, unsigned offset) { struct pic32_gpio_bank *bank = gpiochip_get_data(chip); - return !!(readl(bank->reg_base + TRIS_REG) & BIT(offset)); + if (readl(bank->reg_base + TRIS_REG) & BIT(offset)) + return GPIO_LINE_DIRECTION_IN; + + return GPIO_LINE_DIRECTION_OUT; } static void pic32_gpio_irq_ack(struct irq_data *data) diff --git a/drivers/pinctrl/pinctrl-pistachio.c b/drivers/pinctrl/pinctrl-pistachio.c index fa370c171cad..ec761ba2a2da 100644 --- a/drivers/pinctrl/pinctrl-pistachio.c +++ b/drivers/pinctrl/pinctrl-pistachio.c @@ -1166,7 +1166,10 @@ static int pistachio_gpio_get_direction(struct gpio_chip *chip, unsigned offset) { struct pistachio_gpio_bank *bank = gpiochip_get_data(chip); - return !(gpio_readl(bank, GPIO_OUTPUT_EN) & BIT(offset)); + if (gpio_readl(bank, GPIO_OUTPUT_EN) & BIT(offset)) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int pistachio_gpio_get(struct gpio_chip *chip, unsigned offset) diff --git a/drivers/pinctrl/pinctrl-rk805.c b/drivers/pinctrl/pinctrl-rk805.c index 26adbe9d6d42..cccbe072274e 100644 --- a/drivers/pinctrl/pinctrl-rk805.c +++ b/drivers/pinctrl/pinctrl-rk805.c @@ -184,7 +184,7 @@ static int rk805_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) /* default output*/ if (!pci->pin_cfg[offset].dir_msk) - return 0; + return GPIO_LINE_DIRECTION_OUT; ret = regmap_read(pci->rk808->regmap, pci->pin_cfg[offset].reg, @@ -194,7 +194,10 @@ static int rk805_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) return ret; } - return !(val & pci->pin_cfg[offset].dir_msk); + if (val & pci->pin_cfg[offset].dir_msk) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static const struct gpio_chip rk805_gpio_chip = { diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c index fc9a2a9959d9..098951346339 100644 --- a/drivers/pinctrl/pinctrl-rockchip.c +++ b/drivers/pinctrl/pinctrl-rockchip.c @@ -2549,7 +2549,10 @@ static int rockchip_gpio_get_direction(struct gpio_chip *chip, unsigned offset) data = readl_relaxed(bank->reg_base + GPIO_SWPORT_DDR); clk_disable(bank->clk); - return !(data & BIT(offset)); + if (data & BIT(offset)) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } /* diff --git a/drivers/pinctrl/pinctrl-rza1.c b/drivers/pinctrl/pinctrl-rza1.c index 617585be6a7d..da2d8365c690 100644 --- a/drivers/pinctrl/pinctrl-rza1.c +++ b/drivers/pinctrl/pinctrl-rza1.c @@ -777,7 +777,10 @@ static int rza1_gpio_get_direction(struct gpio_chip *chip, unsigned int gpio) { struct rza1_port *port = gpiochip_get_data(chip); - return !!rza1_get_bit(port, RZA1_PM_REG, gpio); + if (rza1_get_bit(port, RZA1_PM_REG, gpio)) + return GPIO_LINE_DIRECTION_IN; + + return GPIO_LINE_DIRECTION_OUT; } static int rza1_gpio_direction_input(struct gpio_chip *chip, diff --git a/drivers/pinctrl/pinctrl-rza2.c b/drivers/pinctrl/pinctrl-rza2.c index a205964e839b..c5bf98c86b2b 100644 --- a/drivers/pinctrl/pinctrl-rza2.c +++ b/drivers/pinctrl/pinctrl-rza2.c @@ -135,10 +135,10 @@ static int rza2_chip_get_direction(struct gpio_chip *chip, unsigned int offset) reg16 = (reg16 >> (pin * 2)) & RZA2_PDR_MASK; if (reg16 == RZA2_PDR_OUTPUT) - return 0; + return GPIO_LINE_DIRECTION_OUT; if (reg16 == RZA2_PDR_INPUT) - return 1; + return GPIO_LINE_DIRECTION_IN; /* * This GPIO controller has a default Hi-Z state that is not input or @@ -146,7 +146,7 @@ static int rza2_chip_get_direction(struct gpio_chip *chip, unsigned int offset) */ rza2_pin_to_gpio(priv->base, offset, 1); - return 1; + return GPIO_LINE_DIRECTION_IN; } static int rza2_chip_direction_input(struct gpio_chip *chip, diff --git a/drivers/pinctrl/pinctrl-st.c b/drivers/pinctrl/pinctrl-st.c index 4f39a7945d01..7b8c7a0b13de 100644 --- a/drivers/pinctrl/pinctrl-st.c +++ b/drivers/pinctrl/pinctrl-st.c @@ -746,7 +746,10 @@ static int st_gpio_get_direction(struct gpio_chip *chip, unsigned offset) function = st_pctl_get_pin_function(&pc, offset); if (function) { st_pinconf_get_direction(&pc, offset, &config); - return !ST_PINCONF_UNPACK_OE(config); + if (ST_PINCONF_UNPACK_OE(config)) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } /* @@ -758,7 +761,10 @@ static int st_gpio_get_direction(struct gpio_chip *chip, unsigned offset) direction |= ((value >> offset) & 0x1) << i; } - return (direction == ST_GPIO_DIRECTION_IN); + if (direction == ST_GPIO_DIRECTION_IN) + return GPIO_LINE_DIRECTION_IN; + + return GPIO_LINE_DIRECTION_OUT; } /* Pinctrl Groups */ @@ -996,6 +1002,7 @@ static void st_pinconf_dbg_show(struct pinctrl_dev *pctldev, unsigned int function; int offset = st_gpio_pin(pin_id); char f[16]; + int oe; mutex_unlock(&pctldev->mutex); pc = st_get_pio_control(pctldev, pin_id); @@ -1008,10 +1015,11 @@ static void st_pinconf_dbg_show(struct pinctrl_dev *pctldev, else snprintf(f, 5, "GPIO"); + oe = st_gpio_get_direction(&pc_to_bank(pc)->gpio_chip, offset); seq_printf(s, "[OE:%d,PU:%ld,OD:%ld]\t%s\n" "\t\t[retime:%ld,invclk:%ld,clknotdat:%ld," "de:%ld,rt-clk:%ld,rt-delay:%ld]", - !st_gpio_get_direction(&pc_to_bank(pc)->gpio_chip, offset), + (oe == GPIO_LINE_DIRECTION_OUT), ST_PINCONF_UNPACK_PU(config), ST_PINCONF_UNPACK_OD(config), f, diff --git a/drivers/pinctrl/pinctrl-stmfx.c b/drivers/pinctrl/pinctrl-stmfx.c index 16723797fa7c..60100b45f5e5 100644 --- a/drivers/pinctrl/pinctrl-stmfx.c +++ b/drivers/pinctrl/pinctrl-stmfx.c @@ -134,10 +134,14 @@ static int stmfx_gpio_get_direction(struct gpio_chip *gc, unsigned int offset) ret = regmap_read(pctl->stmfx->map, reg, &val); /* * On stmfx, gpio pins direction is (0)input, (1)output. - * .get_direction returns 0=out, 1=in */ + if (ret) + return ret; - return ret ? ret : !(val & mask); + if (val & mask) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int stmfx_gpio_direction_input(struct gpio_chip *gc, unsigned int offset) @@ -223,6 +227,13 @@ static int stmfx_pinconf_get(struct pinctrl_dev *pctldev, dir = stmfx_gpio_get_direction(&pctl->gpio_chip, pin); if (dir < 0) return dir; + + /* + * Currently the gpiolib IN is 1 and OUT is 0 but let's not count + * on it just to be on the safe side also in the future :) + */ + dir = (dir == GPIO_LINE_DIRECTION_IN) ? 1 : 0; + type = stmfx_pinconf_get_type(pctl, pin); if (type < 0) return type; @@ -360,7 +371,7 @@ static void stmfx_pinconf_dbg_show(struct pinctrl_dev *pctldev, if (val < 0) return; - if (!dir) { + if (dir == GPIO_LINE_DIRECTION_OUT) { seq_printf(s, "output %s ", val ? "high" : "low"); if (type) seq_printf(s, "open drain %s internal pull-up ", diff --git a/drivers/pinctrl/pinctrl-sx150x.c b/drivers/pinctrl/pinctrl-sx150x.c index 566665931a04..6e74bd87d959 100644 --- a/drivers/pinctrl/pinctrl-sx150x.c +++ b/drivers/pinctrl/pinctrl-sx150x.c @@ -391,13 +391,16 @@ static int sx150x_gpio_get_direction(struct gpio_chip *chip, int ret; if (sx150x_pin_is_oscio(pctl, offset)) - return false; + return GPIO_LINE_DIRECTION_OUT; ret = regmap_read(pctl->regmap, pctl->data->reg_dir, &value); if (ret < 0) return ret; - return !!(value & BIT(offset)); + if (value & BIT(offset)) + return GPIO_LINE_DIRECTION_IN; + + return GPIO_LINE_DIRECTION_OUT; } static int sx150x_gpio_get(struct gpio_chip *chip, unsigned int offset) @@ -687,7 +690,7 @@ static int sx150x_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin, if (ret < 0) return ret; - if (ret) + if (ret == GPIO_LINE_DIRECTION_IN) return -EINVAL; ret = sx150x_gpio_get(&pctl->gpio, pin); diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig index 811af2f81c39..c5d4428f1f94 100644 --- a/drivers/pinctrl/qcom/Kconfig +++ b/drivers/pinctrl/qcom/Kconfig @@ -50,6 +50,16 @@ config PINCTRL_IPQ8074 Qualcomm Technologies Inc. IPQ8074 platform. Select this for IPQ8074. +config PINCTRL_IPQ6018 + tristate "Qualcomm Technologies, Inc. IPQ6018 pin controller driver" + depends on GPIOLIB && OF + select PINCTRL_MSM + help + This is the pinctrl, pinmux, pinconf and gpiolib driver for + the Qualcomm Technologies Inc. TLMM block found on the + Qualcomm Technologies Inc. IPQ6018 platform. Select this for + IPQ6018. + config PINCTRL_MSM8660 tristate "Qualcomm 8660 pin controller driver" depends on GPIOLIB && OF diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile index c2c2f9ad6827..d9e09045a776 100644 --- a/drivers/pinctrl/qcom/Makefile +++ b/drivers/pinctrl/qcom/Makefile @@ -6,6 +6,7 @@ obj-$(CONFIG_PINCTRL_APQ8084) += pinctrl-apq8084.o obj-$(CONFIG_PINCTRL_IPQ4019) += pinctrl-ipq4019.o obj-$(CONFIG_PINCTRL_IPQ8064) += pinctrl-ipq8064.o obj-$(CONFIG_PINCTRL_IPQ8074) += pinctrl-ipq8074.o +obj-$(CONFIG_PINCTRL_IPQ6018) += pinctrl-ipq6018.o obj-$(CONFIG_PINCTRL_MSM8660) += pinctrl-msm8660.o obj-$(CONFIG_PINCTRL_MSM8960) += pinctrl-msm8960.o obj-$(CONFIG_PINCTRL_MSM8X74) += pinctrl-msm8x74.o diff --git a/drivers/pinctrl/qcom/pinctrl-ipq6018.c b/drivers/pinctrl/qcom/pinctrl-ipq6018.c new file mode 100644 index 000000000000..38c33a778cb8 --- /dev/null +++ b/drivers/pinctrl/qcom/pinctrl-ipq6018.c @@ -0,0 +1,1107 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include + +#include "pinctrl-msm.h" + +#define FUNCTION(fname) \ + [msm_mux_##fname] = { \ + .name = #fname, \ + .groups = fname##_groups, \ + .ngroups = ARRAY_SIZE(fname##_groups), \ + } + +#define REG_SIZE 0x1000 +#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9) \ + { \ + .name = "gpio" #id, \ + .pins = gpio##id##_pins, \ + .npins = (unsigned int)ARRAY_SIZE(gpio##id##_pins), \ + .funcs = (int[]){ \ + msm_mux_gpio, /* gpio mode */ \ + msm_mux_##f1, \ + msm_mux_##f2, \ + msm_mux_##f3, \ + msm_mux_##f4, \ + msm_mux_##f5, \ + msm_mux_##f6, \ + msm_mux_##f7, \ + msm_mux_##f8, \ + msm_mux_##f9 \ + }, \ + .nfuncs = 10, \ + .ctl_reg = REG_SIZE * id, \ + .io_reg = 0x4 + REG_SIZE * id, \ + .intr_cfg_reg = 0x8 + REG_SIZE * id, \ + .intr_status_reg = 0xc + REG_SIZE * id, \ + .intr_target_reg = 0x8 + REG_SIZE * id, \ + .mux_bit = 2, \ + .pull_bit = 0, \ + .drv_bit = 6, \ + .oe_bit = 9, \ + .in_bit = 0, \ + .out_bit = 1, \ + .intr_enable_bit = 0, \ + .intr_status_bit = 0, \ + .intr_target_bit = 5, \ + .intr_target_kpss_val = 3, \ + .intr_raw_status_bit = 4, \ + .intr_polarity_bit = 1, \ + .intr_detection_bit = 2, \ + .intr_detection_width = 2, \ + } + +static const struct pinctrl_pin_desc ipq6018_pins[] = { + PINCTRL_PIN(0, "GPIO_0"), + PINCTRL_PIN(1, "GPIO_1"), + PINCTRL_PIN(2, "GPIO_2"), + PINCTRL_PIN(3, "GPIO_3"), + PINCTRL_PIN(4, "GPIO_4"), + PINCTRL_PIN(5, "GPIO_5"), + PINCTRL_PIN(6, "GPIO_6"), + PINCTRL_PIN(7, "GPIO_7"), + PINCTRL_PIN(8, "GPIO_8"), + PINCTRL_PIN(9, "GPIO_9"), + PINCTRL_PIN(10, "GPIO_10"), + PINCTRL_PIN(11, "GPIO_11"), + PINCTRL_PIN(12, "GPIO_12"), + PINCTRL_PIN(13, "GPIO_13"), + PINCTRL_PIN(14, "GPIO_14"), + PINCTRL_PIN(15, "GPIO_15"), + PINCTRL_PIN(16, "GPIO_16"), + PINCTRL_PIN(17, "GPIO_17"), + PINCTRL_PIN(18, "GPIO_18"), + PINCTRL_PIN(19, "GPIO_19"), + PINCTRL_PIN(20, "GPIO_20"), + PINCTRL_PIN(21, "GPIO_21"), + PINCTRL_PIN(22, "GPIO_22"), + PINCTRL_PIN(23, "GPIO_23"), + PINCTRL_PIN(24, "GPIO_24"), + PINCTRL_PIN(25, "GPIO_25"), + PINCTRL_PIN(26, "GPIO_26"), + PINCTRL_PIN(27, "GPIO_27"), + PINCTRL_PIN(28, "GPIO_28"), + PINCTRL_PIN(29, "GPIO_29"), + PINCTRL_PIN(30, "GPIO_30"), + PINCTRL_PIN(31, "GPIO_31"), + PINCTRL_PIN(32, "GPIO_32"), + PINCTRL_PIN(33, "GPIO_33"), + PINCTRL_PIN(34, "GPIO_34"), + PINCTRL_PIN(35, "GPIO_35"), + PINCTRL_PIN(36, "GPIO_36"), + PINCTRL_PIN(37, "GPIO_37"), + PINCTRL_PIN(38, "GPIO_38"), + PINCTRL_PIN(39, "GPIO_39"), + PINCTRL_PIN(40, "GPIO_40"), + PINCTRL_PIN(41, "GPIO_41"), + PINCTRL_PIN(42, "GPIO_42"), + PINCTRL_PIN(43, "GPIO_43"), + PINCTRL_PIN(44, "GPIO_44"), + PINCTRL_PIN(45, "GPIO_45"), + PINCTRL_PIN(46, "GPIO_46"), + PINCTRL_PIN(47, "GPIO_47"), + PINCTRL_PIN(48, "GPIO_48"), + PINCTRL_PIN(49, "GPIO_49"), + PINCTRL_PIN(50, "GPIO_50"), + PINCTRL_PIN(51, "GPIO_51"), + PINCTRL_PIN(52, "GPIO_52"), + PINCTRL_PIN(53, "GPIO_53"), + PINCTRL_PIN(54, "GPIO_54"), + PINCTRL_PIN(55, "GPIO_55"), + PINCTRL_PIN(56, "GPIO_56"), + PINCTRL_PIN(57, "GPIO_57"), + PINCTRL_PIN(58, "GPIO_58"), + PINCTRL_PIN(59, "GPIO_59"), + PINCTRL_PIN(60, "GPIO_60"), + PINCTRL_PIN(61, "GPIO_61"), + PINCTRL_PIN(62, "GPIO_62"), + PINCTRL_PIN(63, "GPIO_63"), + PINCTRL_PIN(64, "GPIO_64"), + PINCTRL_PIN(65, "GPIO_65"), + PINCTRL_PIN(66, "GPIO_66"), + PINCTRL_PIN(67, "GPIO_67"), + PINCTRL_PIN(68, "GPIO_68"), + PINCTRL_PIN(69, "GPIO_69"), + PINCTRL_PIN(70, "GPIO_70"), + PINCTRL_PIN(71, "GPIO_71"), + PINCTRL_PIN(72, "GPIO_72"), + PINCTRL_PIN(73, "GPIO_73"), + PINCTRL_PIN(74, "GPIO_74"), + PINCTRL_PIN(75, "GPIO_75"), + PINCTRL_PIN(76, "GPIO_76"), + PINCTRL_PIN(77, "GPIO_77"), + PINCTRL_PIN(78, "GPIO_78"), + PINCTRL_PIN(79, "GPIO_79"), +}; + +#define DECLARE_MSM_GPIO_PINS(pin) \ + static const unsigned int gpio##pin##_pins[] = { pin } +DECLARE_MSM_GPIO_PINS(0); +DECLARE_MSM_GPIO_PINS(1); +DECLARE_MSM_GPIO_PINS(2); +DECLARE_MSM_GPIO_PINS(3); +DECLARE_MSM_GPIO_PINS(4); +DECLARE_MSM_GPIO_PINS(5); +DECLARE_MSM_GPIO_PINS(6); +DECLARE_MSM_GPIO_PINS(7); +DECLARE_MSM_GPIO_PINS(8); +DECLARE_MSM_GPIO_PINS(9); +DECLARE_MSM_GPIO_PINS(10); +DECLARE_MSM_GPIO_PINS(11); +DECLARE_MSM_GPIO_PINS(12); +DECLARE_MSM_GPIO_PINS(13); +DECLARE_MSM_GPIO_PINS(14); +DECLARE_MSM_GPIO_PINS(15); +DECLARE_MSM_GPIO_PINS(16); +DECLARE_MSM_GPIO_PINS(17); +DECLARE_MSM_GPIO_PINS(18); +DECLARE_MSM_GPIO_PINS(19); +DECLARE_MSM_GPIO_PINS(20); +DECLARE_MSM_GPIO_PINS(21); +DECLARE_MSM_GPIO_PINS(22); +DECLARE_MSM_GPIO_PINS(23); +DECLARE_MSM_GPIO_PINS(24); +DECLARE_MSM_GPIO_PINS(25); +DECLARE_MSM_GPIO_PINS(26); +DECLARE_MSM_GPIO_PINS(27); +DECLARE_MSM_GPIO_PINS(28); +DECLARE_MSM_GPIO_PINS(29); +DECLARE_MSM_GPIO_PINS(30); +DECLARE_MSM_GPIO_PINS(31); +DECLARE_MSM_GPIO_PINS(32); +DECLARE_MSM_GPIO_PINS(33); +DECLARE_MSM_GPIO_PINS(34); +DECLARE_MSM_GPIO_PINS(35); +DECLARE_MSM_GPIO_PINS(36); +DECLARE_MSM_GPIO_PINS(37); +DECLARE_MSM_GPIO_PINS(38); +DECLARE_MSM_GPIO_PINS(39); +DECLARE_MSM_GPIO_PINS(40); +DECLARE_MSM_GPIO_PINS(41); +DECLARE_MSM_GPIO_PINS(42); +DECLARE_MSM_GPIO_PINS(43); +DECLARE_MSM_GPIO_PINS(44); +DECLARE_MSM_GPIO_PINS(45); +DECLARE_MSM_GPIO_PINS(46); +DECLARE_MSM_GPIO_PINS(47); +DECLARE_MSM_GPIO_PINS(48); +DECLARE_MSM_GPIO_PINS(49); +DECLARE_MSM_GPIO_PINS(50); +DECLARE_MSM_GPIO_PINS(51); +DECLARE_MSM_GPIO_PINS(52); +DECLARE_MSM_GPIO_PINS(53); +DECLARE_MSM_GPIO_PINS(54); +DECLARE_MSM_GPIO_PINS(55); +DECLARE_MSM_GPIO_PINS(56); +DECLARE_MSM_GPIO_PINS(57); +DECLARE_MSM_GPIO_PINS(58); +DECLARE_MSM_GPIO_PINS(59); +DECLARE_MSM_GPIO_PINS(60); +DECLARE_MSM_GPIO_PINS(61); +DECLARE_MSM_GPIO_PINS(62); +DECLARE_MSM_GPIO_PINS(63); +DECLARE_MSM_GPIO_PINS(64); +DECLARE_MSM_GPIO_PINS(65); +DECLARE_MSM_GPIO_PINS(66); +DECLARE_MSM_GPIO_PINS(67); +DECLARE_MSM_GPIO_PINS(68); +DECLARE_MSM_GPIO_PINS(69); +DECLARE_MSM_GPIO_PINS(70); +DECLARE_MSM_GPIO_PINS(71); +DECLARE_MSM_GPIO_PINS(72); +DECLARE_MSM_GPIO_PINS(73); +DECLARE_MSM_GPIO_PINS(74); +DECLARE_MSM_GPIO_PINS(75); +DECLARE_MSM_GPIO_PINS(76); +DECLARE_MSM_GPIO_PINS(77); +DECLARE_MSM_GPIO_PINS(78); +DECLARE_MSM_GPIO_PINS(79); + +enum ipq6018_functions { + msm_mux_atest_char, + msm_mux_atest_char0, + msm_mux_atest_char1, + msm_mux_atest_char2, + msm_mux_atest_char3, + msm_mux_audio0, + msm_mux_audio1, + msm_mux_audio2, + msm_mux_audio3, + msm_mux_audio_rxbclk, + msm_mux_audio_rxfsync, + msm_mux_audio_rxmclk, + msm_mux_audio_rxmclkin, + msm_mux_audio_txbclk, + msm_mux_audio_txfsync, + msm_mux_audio_txmclk, + msm_mux_audio_txmclkin, + msm_mux_blsp0_i2c, + msm_mux_blsp0_spi, + msm_mux_blsp0_uart, + msm_mux_blsp1_i2c, + msm_mux_blsp1_spi, + msm_mux_blsp1_uart, + msm_mux_blsp2_i2c, + msm_mux_blsp2_spi, + msm_mux_blsp2_uart, + msm_mux_blsp3_i2c, + msm_mux_blsp3_spi, + msm_mux_blsp3_uart, + msm_mux_blsp4_i2c, + msm_mux_blsp4_spi, + msm_mux_blsp4_uart, + msm_mux_blsp5_i2c, + msm_mux_blsp5_uart, + msm_mux_burn0, + msm_mux_burn1, + msm_mux_cri_trng, + msm_mux_cri_trng0, + msm_mux_cri_trng1, + msm_mux_cxc0, + msm_mux_cxc1, + msm_mux_dbg_out, + msm_mux_gcc_plltest, + msm_mux_gcc_tlmm, + msm_mux_gpio, + msm_mux_lpass_aud, + msm_mux_lpass_aud0, + msm_mux_lpass_aud1, + msm_mux_lpass_aud2, + msm_mux_lpass_pcm, + msm_mux_lpass_pdm, + msm_mux_mac00, + msm_mux_mac01, + msm_mux_mac10, + msm_mux_mac11, + msm_mux_mac12, + msm_mux_mac13, + msm_mux_mac20, + msm_mux_mac21, + msm_mux_mdc, + msm_mux_mdio, + msm_mux_pcie0_clk, + msm_mux_pcie0_rst, + msm_mux_pcie0_wake, + msm_mux_prng_rosc, + msm_mux_pta1_0, + msm_mux_pta1_1, + msm_mux_pta1_2, + msm_mux_pta2_0, + msm_mux_pta2_1, + msm_mux_pta2_2, + msm_mux_pwm00, + msm_mux_pwm01, + msm_mux_pwm02, + msm_mux_pwm03, + msm_mux_pwm04, + msm_mux_pwm10, + msm_mux_pwm11, + msm_mux_pwm12, + msm_mux_pwm13, + msm_mux_pwm14, + msm_mux_pwm20, + msm_mux_pwm21, + msm_mux_pwm22, + msm_mux_pwm23, + msm_mux_pwm24, + msm_mux_pwm30, + msm_mux_pwm31, + msm_mux_pwm32, + msm_mux_pwm33, + msm_mux_qdss_cti_trig_in_a0, + msm_mux_qdss_cti_trig_in_a1, + msm_mux_qdss_cti_trig_out_a0, + msm_mux_qdss_cti_trig_out_a1, + msm_mux_qdss_cti_trig_in_b0, + msm_mux_qdss_cti_trig_in_b1, + msm_mux_qdss_cti_trig_out_b0, + msm_mux_qdss_cti_trig_out_b1, + msm_mux_qdss_traceclk_a, + msm_mux_qdss_tracectl_a, + msm_mux_qdss_tracedata_a, + msm_mux_qdss_traceclk_b, + msm_mux_qdss_tracectl_b, + msm_mux_qdss_tracedata_b, + msm_mux_qpic_pad, + msm_mux_rx0, + msm_mux_rx1, + msm_mux_rx_swrm, + msm_mux_rx_swrm0, + msm_mux_rx_swrm1, + msm_mux_sd_card, + msm_mux_sd_write, + msm_mux_tsens_max, + msm_mux_tx_swrm, + msm_mux_tx_swrm0, + msm_mux_tx_swrm1, + msm_mux_tx_swrm2, + msm_mux_wci20, + msm_mux_wci21, + msm_mux_wci22, + msm_mux_wci23, + msm_mux_wsa_swrm, + msm_mux__, +}; + +static const char * const blsp3_uart_groups[] = { + "gpio73", "gpio74", "gpio75", "gpio76", +}; + +static const char * const blsp3_i2c_groups[] = { + "gpio73", "gpio74", +}; + +static const char * const blsp3_spi_groups[] = { + "gpio73", "gpio74", "gpio75", "gpio76", "gpio77", "gpio78", "gpio79", +}; + +static const char * const wci20_groups[] = { + "gpio0", "gpio2", +}; + +static const char * const qpic_pad_groups[] = { + "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio9", "gpio10", + "gpio11", "gpio17", +}; + +static const char * const burn0_groups[] = { + "gpio0", +}; + +static const char * const mac12_groups[] = { + "gpio1", "gpio11", +}; + +static const char * const qdss_tracectl_b_groups[] = { + "gpio1", +}; + +static const char * const burn1_groups[] = { + "gpio1", +}; + +static const char * const qdss_traceclk_b_groups[] = { + "gpio0", +}; + +static const char * const qdss_tracedata_b_groups[] = { + "gpio2", "gpio3", "gpio4", "gpio5", "gpio6", "gpio7", "gpio8", "gpio9", + "gpio10", "gpio11", "gpio12", "gpio13", "gpio14", "gpio15", "gpio16", + "gpio17", +}; + +static const char * const mac01_groups[] = { + "gpio3", "gpio4", +}; + +static const char * const mac21_groups[] = { + "gpio5", "gpio6", +}; + +static const char * const atest_char_groups[] = { + "gpio9", +}; + +static const char * const cxc0_groups[] = { + "gpio9", "gpio16", +}; + +static const char * const mac13_groups[] = { + "gpio9", "gpio16", +}; + +static const char * const dbg_out_groups[] = { + "gpio9", +}; + +static const char * const wci22_groups[] = { + "gpio11", "gpio17", +}; + +static const char * const pwm00_groups[] = { + "gpio18", +}; + +static const char * const atest_char0_groups[] = { + "gpio18", +}; + +static const char * const wci23_groups[] = { + "gpio18", "gpio19", +}; + +static const char * const mac11_groups[] = { + "gpio18", "gpio19", +}; + +static const char * const pwm10_groups[] = { + "gpio19", +}; + +static const char * const atest_char1_groups[] = { + "gpio19", +}; + +static const char * const pwm20_groups[] = { + "gpio20", +}; + +static const char * const atest_char2_groups[] = { + "gpio20", +}; + +static const char * const pwm30_groups[] = { + "gpio21", +}; + +static const char * const atest_char3_groups[] = { + "gpio21", +}; + +static const char * const audio_txmclk_groups[] = { + "gpio22", +}; + +static const char * const audio_txmclkin_groups[] = { + "gpio22", +}; + +static const char * const pwm02_groups[] = { + "gpio22", +}; + +static const char * const tx_swrm0_groups[] = { + "gpio22", +}; + +static const char * const qdss_cti_trig_out_b0_groups[] = { + "gpio22", +}; + +static const char * const audio_txbclk_groups[] = { + "gpio23", +}; + +static const char * const pwm12_groups[] = { + "gpio23", +}; + +static const char * const wsa_swrm_groups[] = { + "gpio23", "gpio24", +}; + +static const char * const tx_swrm1_groups[] = { + "gpio23", +}; + +static const char * const qdss_cti_trig_in_b0_groups[] = { + "gpio23", +}; + +static const char * const audio_txfsync_groups[] = { + "gpio24", +}; + +static const char * const pwm22_groups[] = { + "gpio24", +}; + +static const char * const tx_swrm2_groups[] = { + "gpio24", +}; + +static const char * const qdss_cti_trig_out_b1_groups[] = { + "gpio24", +}; + +static const char * const audio0_groups[] = { + "gpio25", "gpio32", +}; + +static const char * const pwm32_groups[] = { + "gpio25", +}; + +static const char * const tx_swrm_groups[] = { + "gpio25", +}; + +static const char * const qdss_cti_trig_in_b1_groups[] = { + "gpio25", +}; + +static const char * const audio1_groups[] = { + "gpio26", "gpio33", +}; + +static const char * const pwm04_groups[] = { + "gpio26", +}; + +static const char * const audio2_groups[] = { + "gpio27", +}; + +static const char * const pwm14_groups[] = { + "gpio27", +}; + +static const char * const audio3_groups[] = { + "gpio28", +}; + +static const char * const pwm24_groups[] = { + "gpio28", +}; + +static const char * const audio_rxmclk_groups[] = { + "gpio29", +}; + +static const char * const audio_rxmclkin_groups[] = { + "gpio29", +}; + +static const char * const pwm03_groups[] = { + "gpio29", +}; + +static const char * const lpass_pdm_groups[] = { + "gpio29", "gpio30", "gpio31", "gpio32", +}; + +static const char * const lpass_aud_groups[] = { + "gpio29", +}; + +static const char * const qdss_cti_trig_in_a1_groups[] = { + "gpio29", +}; + +static const char * const audio_rxbclk_groups[] = { + "gpio30", +}; + +static const char * const pwm13_groups[] = { + "gpio30", +}; + +static const char * const lpass_aud0_groups[] = { + "gpio30", +}; + +static const char * const rx_swrm_groups[] = { + "gpio30", +}; + +static const char * const qdss_cti_trig_out_a1_groups[] = { + "gpio30", +}; + +static const char * const audio_rxfsync_groups[] = { + "gpio31", +}; + +static const char * const pwm23_groups[] = { + "gpio31", +}; + +static const char * const lpass_aud1_groups[] = { + "gpio31", +}; + +static const char * const rx_swrm0_groups[] = { + "gpio31", +}; + +static const char * const qdss_cti_trig_in_a0_groups[] = { + "gpio31", +}; + +static const char * const pwm33_groups[] = { + "gpio32", +}; + +static const char * const lpass_aud2_groups[] = { + "gpio32", +}; + +static const char * const rx_swrm1_groups[] = { + "gpio32", +}; + +static const char * const qdss_cti_trig_out_a0_groups[] = { + "gpio32", +}; + +static const char * const lpass_pcm_groups[] = { + "gpio34", "gpio35", "gpio36", "gpio37", +}; + +static const char * const mac10_groups[] = { + "gpio34", "gpio35", +}; + +static const char * const mac00_groups[] = { + "gpio34", "gpio35", +}; + +static const char * const mac20_groups[] = { + "gpio36", "gpio37", +}; + +static const char * const blsp0_uart_groups[] = { + "gpio38", "gpio39", "gpio40", "gpio41", +}; + +static const char * const blsp0_i2c_groups[] = { + "gpio38", "gpio39", +}; + +static const char * const blsp0_spi_groups[] = { + "gpio38", "gpio39", "gpio40", "gpio41", +}; + +static const char * const blsp2_uart_groups[] = { + "gpio42", "gpio43", "gpio44", "gpio45", +}; + +static const char * const blsp2_i2c_groups[] = { + "gpio42", "gpio43", +}; + +static const char * const blsp2_spi_groups[] = { + "gpio42", "gpio43", "gpio44", "gpio45", +}; + +static const char * const blsp5_i2c_groups[] = { + "gpio46", "gpio47", +}; + +static const char * const blsp5_uart_groups[] = { + "gpio48", "gpio49", +}; + +static const char * const qdss_traceclk_a_groups[] = { + "gpio48", +}; + +static const char * const qdss_tracectl_a_groups[] = { + "gpio49", +}; + +static const char * const pwm01_groups[] = { + "gpio50", +}; + +static const char * const pta1_1_groups[] = { + "gpio51", +}; + +static const char * const pwm11_groups[] = { + "gpio51", +}; + +static const char * const rx1_groups[] = { + "gpio51", +}; + +static const char * const pta1_2_groups[] = { + "gpio52", +}; + +static const char * const pwm21_groups[] = { + "gpio52", +}; + +static const char * const pta1_0_groups[] = { + "gpio53", +}; + +static const char * const pwm31_groups[] = { + "gpio53", +}; + +static const char * const prng_rosc_groups[] = { + "gpio53", +}; + +static const char * const blsp4_uart_groups[] = { + "gpio55", "gpio56", "gpio57", "gpio58", +}; + +static const char * const blsp4_i2c_groups[] = { + "gpio55", "gpio56", +}; + +static const char * const blsp4_spi_groups[] = { + "gpio55", "gpio56", "gpio57", "gpio58", +}; + +static const char * const pcie0_clk_groups[] = { + "gpio59", +}; + +static const char * const cri_trng0_groups[] = { + "gpio59", +}; + +static const char * const pcie0_rst_groups[] = { + "gpio60", +}; + +static const char * const cri_trng1_groups[] = { + "gpio60", +}; + +static const char * const pcie0_wake_groups[] = { + "gpio61", +}; + +static const char * const cri_trng_groups[] = { + "gpio61", +}; + +static const char * const sd_card_groups[] = { + "gpio62", +}; + +static const char * const sd_write_groups[] = { + "gpio63", +}; + +static const char * const rx0_groups[] = { + "gpio63", +}; + +static const char * const tsens_max_groups[] = { + "gpio63", +}; + +static const char * const mdc_groups[] = { + "gpio64", +}; + +static const char * const qdss_tracedata_a_groups[] = { + "gpio64", "gpio65", "gpio66", "gpio67", "gpio68", "gpio69", "gpio70", + "gpio71", "gpio72", "gpio73", "gpio74", "gpio75", "gpio76", "gpio77", + "gpio78", "gpio79", +}; + +static const char * const mdio_groups[] = { + "gpio65", +}; + +static const char * const pta2_0_groups[] = { + "gpio66", +}; + +static const char * const wci21_groups[] = { + "gpio66", "gpio68", +}; + +static const char * const cxc1_groups[] = { + "gpio66", "gpio68", +}; + +static const char * const pta2_1_groups[] = { + "gpio67", +}; + +static const char * const pta2_2_groups[] = { + "gpio68", +}; + +static const char * const blsp1_uart_groups[] = { + "gpio69", "gpio70", "gpio71", "gpio72", +}; + +static const char * const blsp1_i2c_groups[] = { + "gpio69", "gpio70", +}; + +static const char * const blsp1_spi_groups[] = { + "gpio69", "gpio70", "gpio71", "gpio72", +}; + +static const char * const gcc_plltest_groups[] = { + "gpio69", "gpio71", +}; + +static const char * const gcc_tlmm_groups[] = { + "gpio70", +}; + +static const char * const gpio_groups[] = { + "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", "gpio6", "gpio7", + "gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio14", + "gpio15", "gpio16", "gpio17", "gpio18", "gpio19", "gpio20", "gpio21", + "gpio22", "gpio23", "gpio24", "gpio25", "gpio26", "gpio27", "gpio28", + "gpio29", "gpio30", "gpio31", "gpio32", "gpio33", "gpio34", "gpio35", + "gpio36", "gpio37", "gpio38", "gpio39", "gpio40", "gpio41", "gpio42", + "gpio43", "gpio44", "gpio45", "gpio46", "gpio47", "gpio48", "gpio49", + "gpio50", "gpio51", "gpio52", "gpio53", "gpio54", "gpio55", "gpio56", + "gpio57", "gpio58", "gpio59", "gpio60", "gpio61", "gpio62", "gpio63", + "gpio64", "gpio65", "gpio66", "gpio67", "gpio68", "gpio69", "gpio70", + "gpio71", "gpio72", "gpio73", "gpio74", "gpio75", "gpio76", "gpio77", + "gpio78", "gpio79", +}; + +static const struct msm_function ipq6018_functions[] = { + FUNCTION(atest_char), + FUNCTION(atest_char0), + FUNCTION(atest_char1), + FUNCTION(atest_char2), + FUNCTION(atest_char3), + FUNCTION(audio0), + FUNCTION(audio1), + FUNCTION(audio2), + FUNCTION(audio3), + FUNCTION(audio_rxbclk), + FUNCTION(audio_rxfsync), + FUNCTION(audio_rxmclk), + FUNCTION(audio_rxmclkin), + FUNCTION(audio_txbclk), + FUNCTION(audio_txfsync), + FUNCTION(audio_txmclk), + FUNCTION(audio_txmclkin), + FUNCTION(blsp0_i2c), + FUNCTION(blsp0_spi), + FUNCTION(blsp0_uart), + FUNCTION(blsp1_i2c), + FUNCTION(blsp1_spi), + FUNCTION(blsp1_uart), + FUNCTION(blsp2_i2c), + FUNCTION(blsp2_spi), + FUNCTION(blsp2_uart), + FUNCTION(blsp3_i2c), + FUNCTION(blsp3_spi), + FUNCTION(blsp3_uart), + FUNCTION(blsp4_i2c), + FUNCTION(blsp4_spi), + FUNCTION(blsp4_uart), + FUNCTION(blsp5_i2c), + FUNCTION(blsp5_uart), + FUNCTION(burn0), + FUNCTION(burn1), + FUNCTION(cri_trng), + FUNCTION(cri_trng0), + FUNCTION(cri_trng1), + FUNCTION(cxc0), + FUNCTION(cxc1), + FUNCTION(dbg_out), + FUNCTION(gcc_plltest), + FUNCTION(gcc_tlmm), + FUNCTION(gpio), + FUNCTION(lpass_aud), + FUNCTION(lpass_aud0), + FUNCTION(lpass_aud1), + FUNCTION(lpass_aud2), + FUNCTION(lpass_pcm), + FUNCTION(lpass_pdm), + FUNCTION(mac00), + FUNCTION(mac01), + FUNCTION(mac10), + FUNCTION(mac11), + FUNCTION(mac12), + FUNCTION(mac13), + FUNCTION(mac20), + FUNCTION(mac21), + FUNCTION(mdc), + FUNCTION(mdio), + FUNCTION(pcie0_clk), + FUNCTION(pcie0_rst), + FUNCTION(pcie0_wake), + FUNCTION(prng_rosc), + FUNCTION(pta1_0), + FUNCTION(pta1_1), + FUNCTION(pta1_2), + FUNCTION(pta2_0), + FUNCTION(pta2_1), + FUNCTION(pta2_2), + FUNCTION(pwm00), + FUNCTION(pwm01), + FUNCTION(pwm02), + FUNCTION(pwm03), + FUNCTION(pwm04), + FUNCTION(pwm10), + FUNCTION(pwm11), + FUNCTION(pwm12), + FUNCTION(pwm13), + FUNCTION(pwm14), + FUNCTION(pwm20), + FUNCTION(pwm21), + FUNCTION(pwm22), + FUNCTION(pwm23), + FUNCTION(pwm24), + FUNCTION(pwm30), + FUNCTION(pwm31), + FUNCTION(pwm32), + FUNCTION(pwm33), + FUNCTION(qdss_cti_trig_in_a0), + FUNCTION(qdss_cti_trig_in_a1), + FUNCTION(qdss_cti_trig_out_a0), + FUNCTION(qdss_cti_trig_out_a1), + FUNCTION(qdss_cti_trig_in_b0), + FUNCTION(qdss_cti_trig_in_b1), + FUNCTION(qdss_cti_trig_out_b0), + FUNCTION(qdss_cti_trig_out_b1), + FUNCTION(qdss_traceclk_a), + FUNCTION(qdss_tracectl_a), + FUNCTION(qdss_tracedata_a), + FUNCTION(qdss_traceclk_b), + FUNCTION(qdss_tracectl_b), + FUNCTION(qdss_tracedata_b), + FUNCTION(qpic_pad), + FUNCTION(rx0), + FUNCTION(rx1), + FUNCTION(rx_swrm), + FUNCTION(rx_swrm0), + FUNCTION(rx_swrm1), + FUNCTION(sd_card), + FUNCTION(sd_write), + FUNCTION(tsens_max), + FUNCTION(tx_swrm), + FUNCTION(tx_swrm0), + FUNCTION(tx_swrm1), + FUNCTION(tx_swrm2), + FUNCTION(wci20), + FUNCTION(wci21), + FUNCTION(wci22), + FUNCTION(wci23), + FUNCTION(wsa_swrm), +}; + +static const struct msm_pingroup ipq6018_groups[] = { + PINGROUP(0, qpic_pad, wci20, qdss_traceclk_b, _, burn0, _, _, _, _), + PINGROUP(1, qpic_pad, mac12, qdss_tracectl_b, _, burn1, _, _, _, _), + PINGROUP(2, qpic_pad, wci20, qdss_tracedata_b, _, _, _, _, _, _), + PINGROUP(3, qpic_pad, mac01, qdss_tracedata_b, _, _, _, _, _, _), + PINGROUP(4, qpic_pad, mac01, qdss_tracedata_b, _, _, _, _, _, _), + PINGROUP(5, qpic_pad, mac21, qdss_tracedata_b, _, _, _, _, _, _), + PINGROUP(6, qpic_pad, mac21, qdss_tracedata_b, _, _, _, _, _, _), + PINGROUP(7, qpic_pad, qdss_tracedata_b, _, _, _, _, _, _, _), + PINGROUP(8, qpic_pad, qdss_tracedata_b, _, _, _, _, _, _, _), + PINGROUP(9, qpic_pad, atest_char, cxc0, mac13, dbg_out, qdss_tracedata_b, _, _, _), + PINGROUP(10, qpic_pad, qdss_tracedata_b, _, _, _, _, _, _, _), + PINGROUP(11, qpic_pad, wci22, mac12, qdss_tracedata_b, _, _, _, _, _), + PINGROUP(12, qpic_pad, qdss_tracedata_b, _, _, _, _, _, _, _), + PINGROUP(13, qpic_pad, qdss_tracedata_b, _, _, _, _, _, _, _), + PINGROUP(14, qpic_pad, qdss_tracedata_b, _, _, _, _, _, _, _), + PINGROUP(15, qpic_pad, qdss_tracedata_b, _, _, _, _, _, _, _), + PINGROUP(16, qpic_pad, cxc0, mac13, qdss_tracedata_b, _, _, _, _, _), + PINGROUP(17, qpic_pad, qdss_tracedata_b, wci22, _, _, _, _, _, _), + PINGROUP(18, pwm00, atest_char0, wci23, mac11, _, _, _, _, _), + PINGROUP(19, pwm10, atest_char1, wci23, mac11, _, _, _, _, _), + PINGROUP(20, pwm20, atest_char2, _, _, _, _, _, _, _), + PINGROUP(21, pwm30, atest_char3, _, _, _, _, _, _, _), + PINGROUP(22, audio_txmclk, audio_txmclkin, pwm02, tx_swrm0, _, qdss_cti_trig_out_b0, _, _, _), + PINGROUP(23, audio_txbclk, pwm12, wsa_swrm, tx_swrm1, _, qdss_cti_trig_in_b0, _, _, _), + PINGROUP(24, audio_txfsync, pwm22, wsa_swrm, tx_swrm2, _, qdss_cti_trig_out_b1, _, _, _), + PINGROUP(25, audio0, pwm32, tx_swrm, _, qdss_cti_trig_in_b1, _, _, _, _), + PINGROUP(26, audio1, pwm04, _, _, _, _, _, _, _), + PINGROUP(27, audio2, pwm14, _, _, _, _, _, _, _), + PINGROUP(28, audio3, pwm24, _, _, _, _, _, _, _), + PINGROUP(29, audio_rxmclk, audio_rxmclkin, pwm03, lpass_pdm, lpass_aud, qdss_cti_trig_in_a1, _, _, _), + PINGROUP(30, audio_rxbclk, pwm13, lpass_pdm, lpass_aud0, rx_swrm, _, qdss_cti_trig_out_a1, _, _), + PINGROUP(31, audio_rxfsync, pwm23, lpass_pdm, lpass_aud1, rx_swrm0, _, qdss_cti_trig_in_a0, _, _), + PINGROUP(32, audio0, pwm33, lpass_pdm, lpass_aud2, rx_swrm1, _, qdss_cti_trig_out_a0, _, _), + PINGROUP(33, audio1, _, _, _, _, _, _, _, _), + PINGROUP(34, lpass_pcm, mac10, mac00, _, _, _, _, _, _), + PINGROUP(35, lpass_pcm, mac10, mac00, _, _, _, _, _, _), + PINGROUP(36, lpass_pcm, mac20, _, _, _, _, _, _, _), + PINGROUP(37, lpass_pcm, mac20, _, _, _, _, _, _, _), + PINGROUP(38, blsp0_uart, blsp0_i2c, blsp0_spi, _, _, _, _, _, _), + PINGROUP(39, blsp0_uart, blsp0_i2c, blsp0_spi, _, _, _, _, _, _), + PINGROUP(40, blsp0_uart, blsp0_spi, _, _, _, _, _, _, _), + PINGROUP(41, blsp0_uart, blsp0_spi, _, _, _, _, _, _, _), + PINGROUP(42, blsp2_uart, blsp2_i2c, blsp2_spi, _, _, _, _, _, _), + PINGROUP(43, blsp2_uart, blsp2_i2c, blsp2_spi, _, _, _, _, _, _), + PINGROUP(44, blsp2_uart, blsp2_spi, _, _, _, _, _, _, _), + PINGROUP(45, blsp2_uart, blsp2_spi, _, _, _, _, _, _, _), + PINGROUP(46, blsp5_i2c, _, _, _, _, _, _, _, _), + PINGROUP(47, blsp5_i2c, _, _, _, _, _, _, _, _), + PINGROUP(48, blsp5_uart, _, qdss_traceclk_a, _, _, _, _, _, _), + PINGROUP(49, blsp5_uart, _, qdss_tracectl_a, _, _, _, _, _, _), + PINGROUP(50, pwm01, _, _, _, _, _, _, _, _), + PINGROUP(51, pta1_1, pwm11, _, rx1, _, _, _, _, _), + PINGROUP(52, pta1_2, pwm21, _, _, _, _, _, _, _), + PINGROUP(53, pta1_0, pwm31, prng_rosc, _, _, _, _, _, _), + PINGROUP(54, _, _, _, _, _, _, _, _, _), + PINGROUP(55, blsp4_uart, blsp4_i2c, blsp4_spi, _, _, _, _, _, _), + PINGROUP(56, blsp4_uart, blsp4_i2c, blsp4_spi, _, _, _, _, _, _), + PINGROUP(57, blsp4_uart, blsp4_spi, _, _, _, _, _, _, _), + PINGROUP(58, blsp4_uart, blsp4_spi, _, _, _, _, _, _, _), + PINGROUP(59, pcie0_clk, _, _, cri_trng0, _, _, _, _, _), + PINGROUP(60, pcie0_rst, _, _, cri_trng1, _, _, _, _, _), + PINGROUP(61, pcie0_wake, _, _, cri_trng, _, _, _, _, _), + PINGROUP(62, sd_card, _, _, _, _, _, _, _, _), + PINGROUP(63, sd_write, rx0, _, tsens_max, _, _, _, _, _), + PINGROUP(64, mdc, _, qdss_tracedata_a, _, _, _, _, _, _), + PINGROUP(65, mdio, _, qdss_tracedata_a, _, _, _, _, _, _), + PINGROUP(66, pta2_0, wci21, cxc1, qdss_tracedata_a, _, _, _, _, _), + PINGROUP(67, pta2_1, qdss_tracedata_a, _, _, _, _, _, _, _), + PINGROUP(68, pta2_2, wci21, cxc1, qdss_tracedata_a, _, _, _, _, _), + PINGROUP(69, blsp1_uart, blsp1_i2c, blsp1_spi, gcc_plltest, qdss_tracedata_a, _, _, _, _), + PINGROUP(70, blsp1_uart, blsp1_i2c, blsp1_spi, gcc_tlmm, qdss_tracedata_a, _, _, _, _), + PINGROUP(71, blsp1_uart, blsp1_spi, gcc_plltest, qdss_tracedata_a, _, _, _, _, _), + PINGROUP(72, blsp1_uart, blsp1_spi, qdss_tracedata_a, _, _, _, _, _, _), + PINGROUP(73, blsp3_uart, blsp3_i2c, blsp3_spi, _, qdss_tracedata_a, _, _, _, _), + PINGROUP(74, blsp3_uart, blsp3_i2c, blsp3_spi, _, qdss_tracedata_a, _, _, _, _), + PINGROUP(75, blsp3_uart, blsp3_spi, _, qdss_tracedata_a, _, _, _, _, _), + PINGROUP(76, blsp3_uart, blsp3_spi, _, qdss_tracedata_a, _, _, _, _, _), + PINGROUP(77, blsp3_spi, _, qdss_tracedata_a, _, _, _, _, _, _), + PINGROUP(78, blsp3_spi, _, qdss_tracedata_a, _, _, _, _, _, _), + PINGROUP(79, blsp3_spi, _, qdss_tracedata_a, _, _, _, _, _, _), +}; + +static const struct msm_pinctrl_soc_data ipq6018_pinctrl = { + .pins = ipq6018_pins, + .npins = ARRAY_SIZE(ipq6018_pins), + .functions = ipq6018_functions, + .nfunctions = ARRAY_SIZE(ipq6018_functions), + .groups = ipq6018_groups, + .ngroups = ARRAY_SIZE(ipq6018_groups), + .ngpios = 80, +}; + +static int ipq6018_pinctrl_probe(struct platform_device *pdev) +{ + return msm_pinctrl_probe(pdev, &ipq6018_pinctrl); +} + +static const struct of_device_id ipq6018_pinctrl_of_match[] = { + { .compatible = "qcom,ipq6018-pinctrl", }, + { }, +}; + +static struct platform_driver ipq6018_pinctrl_driver = { + .driver = { + .name = "ipq6018-pinctrl", + .of_match_table = ipq6018_pinctrl_of_match, + }, + .probe = ipq6018_pinctrl_probe, + .remove = msm_pinctrl_remove, +}; + +static int __init ipq6018_pinctrl_init(void) +{ + return platform_driver_register(&ipq6018_pinctrl_driver); +} +arch_initcall(ipq6018_pinctrl_init); + +static void __exit ipq6018_pinctrl_exit(void) +{ + platform_driver_unregister(&ipq6018_pinctrl_driver); +} +module_exit(ipq6018_pinctrl_exit); + +MODULE_DESCRIPTION("QTI ipq6018 pinctrl driver"); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(of, ipq6018_pinctrl_of_match); diff --git a/drivers/pinctrl/qcom/pinctrl-ipq8064.c b/drivers/pinctrl/qcom/pinctrl-ipq8064.c index c2fb1ddf2f22..ac717ee38416 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq8064.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq8064.c @@ -299,7 +299,7 @@ static const char * const gpio_groups[] = { }; static const char * const mdio_groups[] = { - "gpio0", "gpio1", "gpio10", "gpio11", + "gpio0", "gpio1", "gpio2", "gpio10", "gpio11", "gpio66", }; static const char * const mi2s_groups[] = { @@ -403,8 +403,8 @@ static const char * const usb2_hsic_groups[] = { }; static const char * const rgmii2_groups[] = { - "gpio27", "gpio28", "gpio29", "gpio30", "gpio31", "gpio32", - "gpio51", "gpio52", "gpio59", "gpio60", "gpio61", "gpio62", + "gpio2", "gpio27", "gpio28", "gpio29", "gpio30", "gpio31", "gpio32", + "gpio51", "gpio52", "gpio59", "gpio60", "gpio61", "gpio62", "gpio66", }; static const char * const sata_groups[] = { @@ -539,7 +539,7 @@ static const struct msm_function ipq8064_functions[] = { static const struct msm_pingroup ipq8064_groups[] = { PINGROUP(0, mdio, NA, NA, NA, NA, NA, NA, NA, NA, NA), PINGROUP(1, mdio, NA, NA, NA, NA, NA, NA, NA, NA, NA), - PINGROUP(2, gsbi5_spi_cs3, NA, NA, NA, NA, NA, NA, NA, NA, NA), + PINGROUP(2, gsbi5_spi_cs3, rgmii2, mdio, NA, NA, NA, NA, NA, NA, NA), PINGROUP(3, pcie1_rst, pcie1_prsnt, pdm, NA, NA, NA, NA, NA, NA, NA), PINGROUP(4, pcie1_pwren_n, pcie1_pwren, NA, NA, NA, NA, NA, NA, NA, NA), PINGROUP(5, pcie1_clk_req, pcie1_pwrflt, NA, NA, NA, NA, NA, NA, NA, NA), @@ -603,7 +603,7 @@ static const struct msm_pingroup ipq8064_groups[] = { PINGROUP(63, pcie3_rst, NA, NA, NA, NA, NA, NA, NA, NA, NA), PINGROUP(64, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), PINGROUP(65, pcie3_clk_req, NA, NA, NA, NA, NA, NA, NA, NA, NA), - PINGROUP(66, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), + PINGROUP(66, rgmii2, mdio, NA, NA, NA, NA, NA, NA, NA, NA), PINGROUP(67, usb2_hsic, NA, NA, NA, NA, NA, NA, NA, NA, NA), PINGROUP(68, usb2_hsic, NA, NA, NA, NA, NA, NA, NA, NA, NA), SDC_PINGROUP(sdc3_clk, 0x204a, 14, 6), diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c index 1a948c3f54b7..9a398a211d30 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm.c +++ b/drivers/pinctrl/qcom/pinctrl-msm.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include @@ -60,6 +62,8 @@ struct msm_pinctrl { struct irq_chip irq_chip; int irq; + bool intr_target_use_scm; + raw_spinlock_t lock; DECLARE_BITMAP(dual_edge_irqs, MAX_NR_GPIO); @@ -68,6 +72,7 @@ struct msm_pinctrl { const struct msm_pinctrl_soc_data *soc; void __iomem *regs[MAX_NR_TILES]; + u32 phys_base[MAX_NR_TILES]; }; #define MSM_ACCESSOR(name) \ @@ -489,8 +494,8 @@ static int msm_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) val = msm_readl_ctl(pctrl, g); - /* 0 = output, 1 = input */ - return val & BIT(g->oe_bit) ? 0 : 1; + return val & BIT(g->oe_bit) ? GPIO_LINE_DIRECTION_OUT : + GPIO_LINE_DIRECTION_IN; } static int msm_gpio_get(struct gpio_chip *chip, unsigned offset) @@ -882,11 +887,30 @@ static int msm_gpio_irq_set_type(struct irq_data *d, unsigned int type) else clear_bit(d->hwirq, pctrl->dual_edge_irqs); - /* Route interrupts to application cpu */ - val = msm_readl_intr_target(pctrl, g); - val &= ~(7 << g->intr_target_bit); - val |= g->intr_target_kpss_val << g->intr_target_bit; - msm_writel_intr_target(val, pctrl, g); + /* Route interrupts to application cpu. + * With intr_target_use_scm interrupts are routed to + * application cpu using scm calls. + */ + if (pctrl->intr_target_use_scm) { + u32 addr = pctrl->phys_base[0] + g->intr_target_reg; + int ret; + + qcom_scm_io_readl(addr, &val); + + val &= ~(7 << g->intr_target_bit); + val |= g->intr_target_kpss_val << g->intr_target_bit; + + ret = qcom_scm_io_writel(addr, val); + if (ret) + dev_err(pctrl->dev, + "Failed routing %lu interrupt to Apps proc", + d->hwirq); + } else { + val = msm_readl_intr_target(pctrl, g); + val &= ~(7 << g->intr_target_bit); + val |= g->intr_target_kpss_val << g->intr_target_bit; + msm_writel_intr_target(val, pctrl, g); + } /* Update configuration for gpio. * RAW_STATUS_EN is left on for all gpio irqs. Due to the @@ -1240,6 +1264,9 @@ int msm_pinctrl_probe(struct platform_device *pdev, pctrl->dev = &pdev->dev; pctrl->soc = soc_data; pctrl->chip = msm_gpio_template; + pctrl->intr_target_use_scm = of_device_is_compatible( + pctrl->dev->of_node, + "qcom,ipq8064-pinctrl"); raw_spin_lock_init(&pctrl->lock); @@ -1252,9 +1279,12 @@ int msm_pinctrl_probe(struct platform_device *pdev, return PTR_ERR(pctrl->regs[i]); } } else { - pctrl->regs[0] = devm_platform_ioremap_resource(pdev, 0); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + pctrl->regs[0] = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(pctrl->regs[0])) return PTR_ERR(pctrl->regs[0]); + + pctrl->phys_base[0] = res->start; } msm_pinctrl_setup_pm_reset(pctrl); diff --git a/drivers/pinctrl/sh-pfc/Kconfig b/drivers/pinctrl/sh-pfc/Kconfig index cf0e0dc42b84..9552851b96f1 100644 --- a/drivers/pinctrl/sh-pfc/Kconfig +++ b/drivers/pinctrl/sh-pfc/Kconfig @@ -26,8 +26,8 @@ config PINCTRL_SH_PFC select PINCTRL_PFC_R8A7792 if ARCH_R8A7792 select PINCTRL_PFC_R8A7793 if ARCH_R8A7793 select PINCTRL_PFC_R8A7794 if ARCH_R8A7794 - select PINCTRL_PFC_R8A77950 if ARCH_R8A77950 || ARCH_R8A7795 - select PINCTRL_PFC_R8A77951 if ARCH_R8A77951 || ARCH_R8A7795 + select PINCTRL_PFC_R8A77950 if ARCH_R8A77950 + select PINCTRL_PFC_R8A77951 if ARCH_R8A77951 select PINCTRL_PFC_R8A77960 if ARCH_R8A77960 select PINCTRL_PFC_R8A77961 if ARCH_R8A77961 select PINCTRL_PFC_R8A77965 if ARCH_R8A77965 diff --git a/drivers/pinctrl/sh-pfc/core.c b/drivers/pinctrl/sh-pfc/core.c index 82209116955b..a2e19efa26e3 100644 --- a/drivers/pinctrl/sh-pfc/core.c +++ b/drivers/pinctrl/sh-pfc/core.c @@ -726,6 +726,27 @@ static int sh_pfc_suspend_init(struct sh_pfc *pfc) { return 0; } #endif /* CONFIG_PM_SLEEP && CONFIG_ARM_PSCI_FW */ #ifdef DEBUG +#define SH_PFC_MAX_REGS 300 +#define SH_PFC_MAX_ENUMS 3000 + +static unsigned int sh_pfc_errors __initdata = 0; +static unsigned int sh_pfc_warnings __initdata = 0; +static u32 *sh_pfc_regs __initdata = NULL; +static u32 sh_pfc_num_regs __initdata = 0; +static u16 *sh_pfc_enums __initdata = NULL; +static u32 sh_pfc_num_enums __initdata = 0; + +#define sh_pfc_err(fmt, ...) \ + do { \ + pr_err("%s: " fmt, drvname, ##__VA_ARGS__); \ + sh_pfc_errors++; \ + } while (0) +#define sh_pfc_warn(fmt, ...) \ + do { \ + pr_warn("%s: " fmt, drvname, ##__VA_ARGS__); \ + sh_pfc_warnings++; \ + } while (0) + static bool __init is0s(const u16 *enum_ids, unsigned int n) { unsigned int i; @@ -737,77 +758,181 @@ static bool __init is0s(const u16 *enum_ids, unsigned int n) return true; } -static unsigned int sh_pfc_errors __initdata = 0; -static unsigned int sh_pfc_warnings __initdata = 0; +static bool __init same_name(const char *a, const char *b) +{ + if (!a || !b) + return false; + + return !strcmp(a, b); +} + +static void __init sh_pfc_check_reg(const char *drvname, u32 reg) +{ + unsigned int i; + + for (i = 0; i < sh_pfc_num_regs; i++) + if (reg == sh_pfc_regs[i]) { + sh_pfc_err("reg 0x%x conflict\n", reg); + return; + } + + if (sh_pfc_num_regs == SH_PFC_MAX_REGS) { + pr_warn_once("%s: Please increase SH_PFC_MAX_REGS\n", drvname); + return; + } + + sh_pfc_regs[sh_pfc_num_regs++] = reg; +} + +static int __init sh_pfc_check_enum(const char *drvname, u16 enum_id) +{ + unsigned int i; + + for (i = 0; i < sh_pfc_num_enums; i++) { + if (enum_id == sh_pfc_enums[i]) + return -EINVAL; + } + + if (sh_pfc_num_enums == SH_PFC_MAX_ENUMS) { + pr_warn_once("%s: Please increase SH_PFC_MAX_ENUMS\n", drvname); + return 0; + } + + sh_pfc_enums[sh_pfc_num_enums++] = enum_id; + return 0; +} + +static void __init sh_pfc_check_reg_enums(const char *drvname, u32 reg, + const u16 *enums, unsigned int n) +{ + unsigned int i; + + for (i = 0; i < n; i++) { + if (enums[i] && sh_pfc_check_enum(drvname, enums[i])) + sh_pfc_err("reg 0x%x enum_id %u conflict\n", reg, + enums[i]); + } +} + +static void __init sh_pfc_check_pin(const struct sh_pfc_soc_info *info, + u32 reg, unsigned int pin) +{ + const char *drvname = info->name; + unsigned int i; + + if (pin == SH_PFC_PIN_NONE) + return; + + for (i = 0; i < info->nr_pins; i++) { + if (pin == info->pins[i].pin) + return; + } + + sh_pfc_err("reg 0x%x: pin %u not found\n", reg, pin); +} static void __init sh_pfc_check_cfg_reg(const char *drvname, const struct pinmux_cfg_reg *cfg_reg) { unsigned int i, n, rw, fw; + sh_pfc_check_reg(drvname, cfg_reg->reg); + if (cfg_reg->field_width) { - /* Checked at build time */ - return; + n = cfg_reg->reg_width / cfg_reg->field_width; + /* Skip field checks (done at build time) */ + goto check_enum_ids; } for (i = 0, n = 0, rw = 0; (fw = cfg_reg->var_field_width[i]); i++) { - if (fw > 3 && is0s(&cfg_reg->enum_ids[n], 1 << fw)) { - pr_warn("%s: reg 0x%x: reserved field [%u:%u] can be split to reduce table size\n", - drvname, cfg_reg->reg, rw, rw + fw - 1); - sh_pfc_warnings++; - } + if (fw > 3 && is0s(&cfg_reg->enum_ids[n], 1 << fw)) + sh_pfc_warn("reg 0x%x: reserved field [%u:%u] can be split to reduce table size\n", + cfg_reg->reg, rw, rw + fw - 1); n += 1 << fw; rw += fw; } - if (rw != cfg_reg->reg_width) { - pr_err("%s: reg 0x%x: var_field_width declares %u instead of %u bits\n", - drvname, cfg_reg->reg, rw, cfg_reg->reg_width); - sh_pfc_errors++; - } + if (rw != cfg_reg->reg_width) + sh_pfc_err("reg 0x%x: var_field_width declares %u instead of %u bits\n", + cfg_reg->reg, rw, cfg_reg->reg_width); - if (n != cfg_reg->nr_enum_ids) { - pr_err("%s: reg 0x%x: enum_ids[] has %u instead of %u values\n", - drvname, cfg_reg->reg, cfg_reg->nr_enum_ids, n); - sh_pfc_errors++; + if (n != cfg_reg->nr_enum_ids) + sh_pfc_err("reg 0x%x: enum_ids[] has %u instead of %u values\n", + cfg_reg->reg, cfg_reg->nr_enum_ids, n); + +check_enum_ids: + sh_pfc_check_reg_enums(drvname, cfg_reg->reg, cfg_reg->enum_ids, n); +} + +static void __init sh_pfc_check_drive_reg(const struct sh_pfc_soc_info *info, + const struct pinmux_drive_reg *drive) +{ + const char *drvname = info->name; + unsigned long seen = 0, mask; + unsigned int i; + + sh_pfc_check_reg(info->name, drive->reg); + for (i = 0; i < ARRAY_SIZE(drive->fields); i++) { + const struct pinmux_drive_reg_field *field = &drive->fields[i]; + + if (!field->pin && !field->offset && !field->size) + continue; + + mask = GENMASK(field->offset + field->size, field->offset); + if (mask & seen) + sh_pfc_err("drive_reg 0x%x: field %u overlap\n", + drive->reg, i); + seen |= mask; + + sh_pfc_check_pin(info, drive->reg, field->pin); } } +static void __init sh_pfc_check_bias_reg(const struct sh_pfc_soc_info *info, + const struct pinmux_bias_reg *bias) +{ + unsigned int i; + + sh_pfc_check_reg(info->name, bias->puen); + if (bias->pud) + sh_pfc_check_reg(info->name, bias->pud); + for (i = 0; i < ARRAY_SIZE(bias->pins); i++) + sh_pfc_check_pin(info, bias->puen, bias->pins[i]); +} + static void __init sh_pfc_check_info(const struct sh_pfc_soc_info *info) { - const struct sh_pfc_function *func; const char *drvname = info->name; unsigned int *refcnts; unsigned int i, j, k; pr_info("Checking %s\n", drvname); + sh_pfc_num_regs = 0; + sh_pfc_num_enums = 0; /* Check pins */ for (i = 0; i < info->nr_pins; i++) { + const struct sh_pfc_pin *pin = &info->pins[i]; + + if (!pin->name) { + sh_pfc_err("empty pin %u\n", i); + continue; + } for (j = 0; j < i; j++) { - if (!strcmp(info->pins[i].name, info->pins[j].name)) { - pr_err("%s: pin %s/%s: name conflict\n", - drvname, info->pins[i].name, - info->pins[j].name); - sh_pfc_errors++; - } + const struct sh_pfc_pin *pin2 = &info->pins[j]; - if (info->pins[i].pin != (u16)-1 && - info->pins[i].pin == info->pins[j].pin) { - pr_err("%s: pin %s/%s: pin %u conflict\n", - drvname, info->pins[i].name, - info->pins[j].name, info->pins[i].pin); - sh_pfc_errors++; - } + if (same_name(pin->name, pin2->name)) + sh_pfc_err("pin %s: name conflict\n", + pin->name); - if (info->pins[i].enum_id && - info->pins[i].enum_id == info->pins[j].enum_id) { - pr_err("%s: pin %s/%s: enum_id %u conflict\n", - drvname, info->pins[i].name, - info->pins[j].name, - info->pins[i].enum_id); - sh_pfc_errors++; - } + if (pin->pin != (u16)-1 && pin->pin == pin2->pin) + sh_pfc_err("pin %s/%s: pin %u conflict\n", + pin->name, pin2->name, pin->pin); + + if (pin->enum_id && pin->enum_id == pin2->enum_id) + sh_pfc_err("pin %s/%s: enum_id %u conflict\n", + pin->name, pin2->name, + pin->enum_id); } } @@ -817,45 +942,49 @@ static void __init sh_pfc_check_info(const struct sh_pfc_soc_info *info) return; for (i = 0; i < info->nr_functions; i++) { - func = &info->functions[i]; + const struct sh_pfc_function *func = &info->functions[i]; + if (!func->name) { - pr_err("%s: empty function %u\n", drvname, i); - sh_pfc_errors++; + sh_pfc_err("empty function %u\n", i); continue; } + for (j = 0; j < i; j++) { + if (same_name(func->name, info->functions[j].name)) + sh_pfc_err("function %s: name conflict\n", + func->name); + } for (j = 0; j < func->nr_groups; j++) { for (k = 0; k < info->nr_groups; k++) { - if (info->groups[k].name && - !strcmp(func->groups[j], - info->groups[k].name)) { + if (same_name(func->groups[j], + info->groups[k].name)) { refcnts[k]++; break; } } - if (k == info->nr_groups) { - pr_err("%s: function %s: group %s not found\n", - drvname, func->name, func->groups[j]); - sh_pfc_errors++; - } + if (k == info->nr_groups) + sh_pfc_err("function %s: group %s not found\n", + func->name, func->groups[j]); } } for (i = 0; i < info->nr_groups; i++) { - if (!info->groups[i].name) { - pr_err("%s: empty group %u\n", drvname, i); - sh_pfc_errors++; + const struct sh_pfc_pin_group *group = &info->groups[i]; + + if (!group->name) { + sh_pfc_err("empty group %u\n", i); continue; } - if (!refcnts[i]) { - pr_err("%s: orphan group %s\n", drvname, - info->groups[i].name); - sh_pfc_errors++; - } else if (refcnts[i] > 1) { - pr_warn("%s: group %s referenced by %u functions\n", - drvname, info->groups[i].name, refcnts[i]); - sh_pfc_warnings++; + for (j = 0; j < i; j++) { + if (same_name(group->name, info->groups[j].name)) + sh_pfc_err("group %s: name conflict\n", + group->name); } + if (!refcnts[i]) + sh_pfc_err("orphan group %s\n", group->name); + else if (refcnts[i] > 1) + sh_pfc_warn("group %s referenced by %u functions\n", + group->name, refcnts[i]); } kfree(refcnts); @@ -863,12 +992,62 @@ static void __init sh_pfc_check_info(const struct sh_pfc_soc_info *info) /* Check config register descriptions */ for (i = 0; info->cfg_regs && info->cfg_regs[i].reg; i++) sh_pfc_check_cfg_reg(drvname, &info->cfg_regs[i]); + + /* Check drive strength registers */ + for (i = 0; info->drive_regs && info->drive_regs[i].reg; i++) + sh_pfc_check_drive_reg(info, &info->drive_regs[i]); + + /* Check bias registers */ + for (i = 0; info->bias_regs && info->bias_regs[i].puen; i++) + sh_pfc_check_bias_reg(info, &info->bias_regs[i]); + + /* Check ioctrl registers */ + for (i = 0; info->ioctrl_regs && info->ioctrl_regs[i].reg; i++) + sh_pfc_check_reg(drvname, info->ioctrl_regs[i].reg); + + /* Check data registers */ + for (i = 0; info->data_regs && info->data_regs[i].reg; i++) { + sh_pfc_check_reg(drvname, info->data_regs[i].reg); + sh_pfc_check_reg_enums(drvname, info->data_regs[i].reg, + info->data_regs[i].enum_ids, + info->data_regs[i].reg_width); + } + +#ifdef CONFIG_PINCTRL_SH_FUNC_GPIO + /* Check function GPIOs */ + for (i = 0; i < info->nr_func_gpios; i++) { + const struct pinmux_func *func = &info->func_gpios[i]; + + if (!func->name) { + sh_pfc_err("empty function gpio %u\n", i); + continue; + } + for (j = 0; j < i; j++) { + if (same_name(func->name, info->func_gpios[j].name)) + sh_pfc_err("func_gpio %s: name conflict\n", + func->name); + } + if (sh_pfc_check_enum(drvname, func->enum_id)) + sh_pfc_err("%s enum_id %u conflict\n", func->name, + func->enum_id); + } +#endif } static void __init sh_pfc_check_driver(const struct platform_driver *pdrv) { unsigned int i; + sh_pfc_regs = kcalloc(SH_PFC_MAX_REGS, sizeof(*sh_pfc_regs), + GFP_KERNEL); + if (!sh_pfc_regs) + return; + + sh_pfc_enums = kcalloc(SH_PFC_MAX_ENUMS, sizeof(*sh_pfc_enums), + GFP_KERNEL); + if (!sh_pfc_enums) + goto free_regs; + pr_warn("Checking builtin pinmux tables\n"); for (i = 0; pdrv->id_table[i].name[0]; i++) @@ -881,6 +1060,10 @@ static void __init sh_pfc_check_driver(const struct platform_driver *pdrv) pr_warn("Detected %u errors and %u warnings\n", sh_pfc_errors, sh_pfc_warnings); + + kfree(sh_pfc_enums); +free_regs: + kfree(sh_pfc_regs); } #else /* !DEBUG */ diff --git a/drivers/pinctrl/sh-pfc/gpio.c b/drivers/pinctrl/sh-pfc/gpio.c index 8213e118aa40..9c6e931ae766 100644 --- a/drivers/pinctrl/sh-pfc/gpio.c +++ b/drivers/pinctrl/sh-pfc/gpio.c @@ -205,14 +205,11 @@ static int gpio_pin_to_irq(struct gpio_chip *gc, unsigned offset) for (k = 0; gpios[k] >= 0; k++) { if (gpios[k] == offset) - goto found; + return pfc->irqs[i]; } } return 0; - -found: - return pfc->irqs[i]; } static int gpio_pin_setup(struct sh_pfc_chip *chip) diff --git a/drivers/pinctrl/sirf/pinctrl-atlas7.c b/drivers/pinctrl/sirf/pinctrl-atlas7.c index b1a9611f46b3..50df9e084414 100644 --- a/drivers/pinctrl/sirf/pinctrl-atlas7.c +++ b/drivers/pinctrl/sirf/pinctrl-atlas7.c @@ -352,7 +352,7 @@ struct atlas7_gpio_chip { int nbank; raw_spinlock_t lock; struct gpio_chip chip; - struct atlas7_gpio_bank banks[0]; + struct atlas7_gpio_bank banks[]; }; /** diff --git a/drivers/pinctrl/sprd/Kconfig b/drivers/pinctrl/sprd/Kconfig index b6c5479b58fb..eef35d01b770 100644 --- a/drivers/pinctrl/sprd/Kconfig +++ b/drivers/pinctrl/sprd/Kconfig @@ -4,9 +4,7 @@ # config PINCTRL_SPRD - bool "Spreadtrum pinctrl driver" - depends on OF - depends on ARCH_SPRD || COMPILE_TEST + tristate select PINMUX select PINCONF select GENERIC_PINCONF @@ -15,7 +13,9 @@ config PINCTRL_SPRD Say Y here to enable Spreadtrum pinctrl driver config PINCTRL_SPRD_SC9860 - bool "Spreadtrum SC9860 pinctrl driver" - depends on PINCTRL_SPRD + tristate "Spreadtrum SC9860 pinctrl driver" + depends on OF + depends on ARCH_SPRD || COMPILE_TEST + select PINCTRL_SPRD help Say Y here to enable Spreadtrum SC9860 pinctrl driver diff --git a/drivers/pinctrl/sprd/pinctrl-sprd.c b/drivers/pinctrl/sprd/pinctrl-sprd.c index 157712ab05a8..48cbf2a2837f 100644 --- a/drivers/pinctrl/sprd/pinctrl-sprd.c +++ b/drivers/pinctrl/sprd/pinctrl-sprd.c @@ -464,9 +464,15 @@ static int sprd_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin_id, case PIN_CONFIG_INPUT_ENABLE: arg = (reg >> SLEEP_INPUT_SHIFT) & SLEEP_INPUT_MASK; break; - case PIN_CONFIG_OUTPUT: + case PIN_CONFIG_OUTPUT_ENABLE: arg = reg & SLEEP_OUTPUT_MASK; break; + case PIN_CONFIG_BIAS_HIGH_IMPEDANCE: + if ((reg & SLEEP_OUTPUT) || (reg & SLEEP_INPUT)) + return -EINVAL; + + arg = 1; + break; case PIN_CONFIG_DRIVE_STRENGTH: arg = (reg >> DRIVE_STRENGTH_SHIFT) & DRIVE_STRENGTH_MASK; @@ -635,13 +641,23 @@ static int sprd_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin_id, shift = SLEEP_INPUT_SHIFT; } break; - case PIN_CONFIG_OUTPUT: + case PIN_CONFIG_OUTPUT_ENABLE: if (is_sleep_config == true) { - val |= SLEEP_OUTPUT; + if (arg > 0) + val |= SLEEP_OUTPUT; + else + val &= ~SLEEP_OUTPUT; + mask = SLEEP_OUTPUT_MASK; shift = SLEEP_OUTPUT_SHIFT; } break; + case PIN_CONFIG_BIAS_HIGH_IMPEDANCE: + if (is_sleep_config == true) { + val = shift = 0; + mask = SLEEP_OUTPUT | SLEEP_INPUT; + } + break; case PIN_CONFIG_DRIVE_STRENGTH: if (arg < 2 || arg > 60) return -EINVAL; @@ -1090,6 +1106,7 @@ int sprd_pinctrl_core_probe(struct platform_device *pdev, return 0; } +EXPORT_SYMBOL_GPL(sprd_pinctrl_core_probe); int sprd_pinctrl_remove(struct platform_device *pdev) { @@ -1098,6 +1115,7 @@ int sprd_pinctrl_remove(struct platform_device *pdev) pinctrl_unregister(sprd_pctl->pctl); return 0; } +EXPORT_SYMBOL_GPL(sprd_pinctrl_remove); void sprd_pinctrl_shutdown(struct platform_device *pdev) { @@ -1112,6 +1130,7 @@ void sprd_pinctrl_shutdown(struct platform_device *pdev) return; pinctrl_select_state(pinctl, state); } +EXPORT_SYMBOL_GPL(sprd_pinctrl_shutdown); MODULE_DESCRIPTION("SPREADTRUM Pin Controller Driver"); MODULE_AUTHOR("Baolin Wang "); diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c index af3b24f26ff2..a657cd829ce6 100644 --- a/drivers/pinctrl/stm32/pinctrl-stm32.c +++ b/drivers/pinctrl/stm32/pinctrl-stm32.c @@ -284,9 +284,9 @@ static int stm32_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) stm32_pmx_get_mode(bank, pin, &mode, &alt); if ((alt == 0) && (mode == 0)) - ret = 1; + ret = GPIO_LINE_DIRECTION_IN; else if ((alt == 0) && (mode == 1)) - ret = 0; + ret = GPIO_LINE_DIRECTION_OUT; else ret = -EINVAL; diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c index b35c3245ab3f..8e792f8e2dc9 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c +++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1058,6 +1059,14 @@ static void sunxi_pinctrl_irq_ack_unmask(struct irq_data *d) sunxi_pinctrl_irq_unmask(d); } +static int sunxi_pinctrl_irq_set_wake(struct irq_data *d, unsigned int on) +{ + struct sunxi_pinctrl *pctl = irq_data_get_irq_chip_data(d); + u8 bank = d->hwirq / IRQ_PER_BANK; + + return irq_set_irq_wake(pctl->irq[bank], on); +} + static struct irq_chip sunxi_pinctrl_edge_irq_chip = { .name = "sunxi_pio_edge", .irq_ack = sunxi_pinctrl_irq_ack, @@ -1066,7 +1075,8 @@ static struct irq_chip sunxi_pinctrl_edge_irq_chip = { .irq_request_resources = sunxi_pinctrl_irq_request_resources, .irq_release_resources = sunxi_pinctrl_irq_release_resources, .irq_set_type = sunxi_pinctrl_irq_set_type, - .flags = IRQCHIP_SKIP_SET_WAKE, + .irq_set_wake = sunxi_pinctrl_irq_set_wake, + .flags = IRQCHIP_MASK_ON_SUSPEND, }; static struct irq_chip sunxi_pinctrl_level_irq_chip = { @@ -1081,7 +1091,9 @@ static struct irq_chip sunxi_pinctrl_level_irq_chip = { .irq_request_resources = sunxi_pinctrl_irq_request_resources, .irq_release_resources = sunxi_pinctrl_irq_release_resources, .irq_set_type = sunxi_pinctrl_irq_set_type, - .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_EOI_THREADED | + .irq_set_wake = sunxi_pinctrl_irq_set_wake, + .flags = IRQCHIP_EOI_THREADED | + IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_EOI_IF_HANDLED, }; diff --git a/drivers/pinctrl/tegra/pinctrl-tegra.c b/drivers/pinctrl/tegra/pinctrl-tegra.c index cefbbb8d1a68..21661f6490d6 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra.c +++ b/drivers/pinctrl/tegra/pinctrl-tegra.c @@ -275,11 +275,57 @@ static int tegra_pinctrl_set_mux(struct pinctrl_dev *pctldev, return 0; } +static int tegra_pinctrl_gpio_request_enable(struct pinctrl_dev *pctldev, + struct pinctrl_gpio_range *range, + unsigned int offset) +{ + struct tegra_pmx *pmx = pinctrl_dev_get_drvdata(pctldev); + const struct tegra_pingroup *group; + u32 value; + + if (!pmx->soc->sfsel_in_mux) + return 0; + + group = &pmx->soc->groups[offset]; + + if (group->mux_reg < 0 || group->sfsel_bit < 0) + return -EINVAL; + + value = pmx_readl(pmx, group->mux_bank, group->mux_reg); + value &= ~BIT(group->sfsel_bit); + pmx_writel(pmx, value, group->mux_bank, group->mux_reg); + + return 0; +} + +static void tegra_pinctrl_gpio_disable_free(struct pinctrl_dev *pctldev, + struct pinctrl_gpio_range *range, + unsigned int offset) +{ + struct tegra_pmx *pmx = pinctrl_dev_get_drvdata(pctldev); + const struct tegra_pingroup *group; + u32 value; + + if (!pmx->soc->sfsel_in_mux) + return; + + group = &pmx->soc->groups[offset]; + + if (group->mux_reg < 0 || group->sfsel_bit < 0) + return; + + value = pmx_readl(pmx, group->mux_bank, group->mux_reg); + value |= BIT(group->sfsel_bit); + pmx_writel(pmx, value, group->mux_bank, group->mux_reg); +} + static const struct pinmux_ops tegra_pinmux_ops = { .get_functions_count = tegra_pinctrl_get_funcs_count, .get_function_name = tegra_pinctrl_get_func_name, .get_function_groups = tegra_pinctrl_get_func_groups, .set_mux = tegra_pinctrl_set_mux, + .gpio_request_enable = tegra_pinctrl_gpio_request_enable, + .gpio_disable_free = tegra_pinctrl_gpio_disable_free, }; static int tegra_pinconf_reg(struct tegra_pmx *pmx, @@ -689,12 +735,12 @@ const struct dev_pm_ops tegra_pinctrl_pm = { .resume = &tegra_pinctrl_resume }; -static bool gpio_node_has_range(const char *compatible) +static bool tegra_pinctrl_gpio_node_has_range(struct tegra_pmx *pmx) { struct device_node *np; bool has_prop = false; - np = of_find_compatible_node(NULL, NULL, compatible); + np = of_find_compatible_node(NULL, NULL, pmx->soc->gpio_compatible); if (!np) return has_prop; @@ -794,7 +840,7 @@ int tegra_pinctrl_probe(struct platform_device *pdev, tegra_pinctrl_clear_parked_bits(pmx); - if (!gpio_node_has_range(pmx->soc->gpio_compatible)) + if (pmx->soc->ngpios > 0 && !tegra_pinctrl_gpio_node_has_range(pmx)) pinctrl_add_gpio_range(pmx->pctl, &tegra_pinctrl_gpio_range); platform_set_drvdata(pdev, pmx); diff --git a/drivers/pinctrl/tegra/pinctrl-tegra.h b/drivers/pinctrl/tegra/pinctrl-tegra.h index 0fc82eea9cf1..fcad7f74c5a2 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra.h +++ b/drivers/pinctrl/tegra/pinctrl-tegra.h @@ -107,7 +107,8 @@ struct tegra_function { * drvup, slwr, slwf, and drvtype parameters. * @drv_bank: Drive fields register bank. * @hsm_bit: High Speed Mode register bit. - * @schmitt_bit: Scmitt register bit. + * @sfsel_bit: GPIO/SFIO selection register bit. + * @schmitt_bit: Schmitt register bit. * @lpmd_bit: Low Power Mode register bit. * @drvdn_bit: Drive Down register bit. * @drvdn_width: Drive Down field width. @@ -153,6 +154,7 @@ struct tegra_pingroup { s32 ioreset_bit:6; s32 rcv_sel_bit:6; s32 hsm_bit:6; + s32 sfsel_bit:6; s32 schmitt_bit:6; s32 lpmd_bit:6; s32 drvdn_bit:6; @@ -192,6 +194,7 @@ struct tegra_pinctrl_soc_data { bool hsm_in_mux; bool schmitt_in_mux; bool drvtype_in_mux; + bool sfsel_in_mux; }; extern const struct dev_pm_ops tegra_pinctrl_pm; diff --git a/drivers/pinctrl/tegra/pinctrl-tegra194.c b/drivers/pinctrl/tegra/pinctrl-tegra194.c index daf44cf240c9..2e0b5f7bb095 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra194.c +++ b/drivers/pinctrl/tegra/pinctrl-tegra194.c @@ -24,17 +24,14 @@ /* Define unique ID for each pins */ enum pin_id { - TEGRA_PIN_PEX_L5_CLKREQ_N_PGG0 = 256, - TEGRA_PIN_PEX_L5_RST_N_PGG1 = 257, - TEGRA_PIN_NUM_GPIOS = 258, + TEGRA_PIN_PEX_L5_CLKREQ_N_PGG0, + TEGRA_PIN_PEX_L5_RST_N_PGG1, }; /* Table for pin descriptor */ static const struct pinctrl_pin_desc tegra194_pins[] = { - PINCTRL_PIN(TEGRA_PIN_PEX_L5_CLKREQ_N_PGG0, - "TEGRA_PIN_PEX_L5_CLKREQ_N_PGG0"), - PINCTRL_PIN(TEGRA_PIN_PEX_L5_RST_N_PGG1, - "TEGRA_PIN_PEX_L5_RST_N_PGG1"), + PINCTRL_PIN(TEGRA_PIN_PEX_L5_CLKREQ_N_PGG0, "PEX_L5_CLKREQ_N_PGG0"), + PINCTRL_PIN(TEGRA_PIN_PEX_L5_RST_N_PGG1, "PEX_L5_RST_N_PGG1"), }; static const unsigned int pex_l5_clkreq_n_pgg0_pins[] = { @@ -59,6 +56,7 @@ enum tegra_mux_dt { { \ .name = #lid, \ } + static struct tegra_function tegra194_functions[] = { TEGRA_PIN_FUNCTION(rsvd0), TEGRA_PIN_FUNCTION(rsvd1), @@ -70,7 +68,7 @@ static struct tegra_function tegra194_functions[] = { #define DRV_PINGROUP_ENTRY_Y(r, drvdn_b, drvdn_w, drvup_b, \ drvup_w, slwr_b, slwr_w, slwf_b, \ slwf_w, bank) \ - .drv_reg = ((r)), \ + .drv_reg = ((r)), \ .drv_bank = bank, \ .drvdn_bit = drvdn_b, \ .drvdn_width = drvdn_w, \ @@ -89,7 +87,7 @@ static struct tegra_function tegra194_functions[] = { .hsm_bit = -1, \ .mux_bank = bank, \ .mux_bit = 0, \ - .pupd_reg = ((r)), \ + .pupd_reg = ((r)), \ .pupd_bank = bank, \ .pupd_bit = 2, \ .tri_reg = ((r)), \ @@ -97,6 +95,7 @@ static struct tegra_function tegra194_functions[] = { .tri_bit = 4, \ .einput_bit = e_input, \ .odrain_bit = e_od, \ + .sfsel_bit = 10, \ .schmitt_bit = schmitt_b, \ .drvtype_bit = 13, \ .drv_reg = -1, \ @@ -109,20 +108,20 @@ static struct tegra_function tegra194_functions[] = { #define PINGROUP(pg_name, f0, f1, f2, f3, r, bank, pupd, e_lpbk, \ e_input, e_lpdr, e_od, schmitt_b, drvtype, io_rail) \ - { \ - .name = #pg_name, \ - .pins = pg_name##_pins, \ - .npins = ARRAY_SIZE(pg_name##_pins), \ - .funcs = { \ - TEGRA_MUX_##f0, \ - TEGRA_MUX_##f1, \ - TEGRA_MUX_##f2, \ - TEGRA_MUX_##f3, \ - }, \ - PIN_PINGROUP_ENTRY_Y(r, bank, pupd, e_lpbk, \ - e_input, e_od, \ - schmitt_b, drvtype), \ - drive_##pg_name, \ + { \ + .name = #pg_name, \ + .pins = pg_name##_pins, \ + .npins = ARRAY_SIZE(pg_name##_pins), \ + .funcs = { \ + TEGRA_MUX_##f0, \ + TEGRA_MUX_##f1, \ + TEGRA_MUX_##f2, \ + TEGRA_MUX_##f3, \ + }, \ + PIN_PINGROUP_ENTRY_Y(r, bank, pupd, e_lpbk, \ + e_input, e_od, \ + schmitt_b, drvtype), \ + drive_##pg_name, \ } static const struct tegra_pingroup tegra194_groups[] = { @@ -133,7 +132,6 @@ static const struct tegra_pingroup tegra194_groups[] = { }; static const struct tegra_pinctrl_soc_data tegra194_pinctrl = { - .ngpios = TEGRA_PIN_NUM_GPIOS, .pins = tegra194_pins, .npins = ARRAY_SIZE(tegra194_pins), .functions = tegra194_functions, @@ -143,6 +141,7 @@ static const struct tegra_pinctrl_soc_data tegra194_pinctrl = { .hsm_in_mux = true, .schmitt_in_mux = true, .drvtype_in_mux = true, + .sfsel_in_mux = true, }; static int tegra194_pinctrl_probe(struct platform_device *pdev) diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-core.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-core.c index 57babf31e320..ade348b49b31 100644 --- a/drivers/pinctrl/uniphier/pinctrl-uniphier-core.c +++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-core.c @@ -29,7 +29,7 @@ struct uniphier_pinctrl_reg_region { struct list_head node; unsigned int base; unsigned int nregs; - u32 vals[0]; + u32 vals[]; }; struct uniphier_pinctrl_priv { diff --git a/drivers/pinctrl/vt8500/pinctrl-wmt.c b/drivers/pinctrl/vt8500/pinctrl-wmt.c index ea910a18b4d7..65b97e240196 100644 --- a/drivers/pinctrl/vt8500/pinctrl-wmt.c +++ b/drivers/pinctrl/vt8500/pinctrl-wmt.c @@ -486,8 +486,10 @@ static int wmt_gpio_get_direction(struct gpio_chip *chip, unsigned offset) u32 val; val = readl_relaxed(data->base + reg_dir); - /* Return 0 == output, 1 == input */ - return !(val & BIT(bit)); + if (val & BIT(bit)) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; } static int wmt_gpio_get_value(struct gpio_chip *chip, unsigned offset) diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index ffdb5bc25d6d..fbaed079b299 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -35,7 +35,7 @@ config MTK_SCP config OMAP_REMOTEPROC tristate "OMAP remoteproc support" - depends on ARCH_OMAP4 || SOC_OMAP5 + depends on ARCH_OMAP4 || SOC_OMAP5 || SOC_DRA7XX depends on OMAP_IOMMU select MAILBOX select OMAP2PLUS_MBOX @@ -52,6 +52,18 @@ config OMAP_REMOTEPROC It's safe to say N here if you're not interested in multimedia offloading or just want a bare minimum kernel. +config OMAP_REMOTEPROC_WATCHDOG + bool "OMAP remoteproc watchdog timer" + depends on OMAP_REMOTEPROC + default n + help + Say Y here to enable watchdog timer for remote processors. + + This option controls the watchdog functionality for the remote + processors in OMAP. Dedicated OMAP DMTimers are used by the remote + processors and triggers the timer interrupt upon a watchdog + detection. + config WKUP_M3_RPROC tristate "AMx3xx Wakeup M3 remoteproc support" depends on SOC_AM33XX || SOC_AM43XX diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 3e72b6f38d4b..8957ed271d20 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -186,7 +186,7 @@ static int imx_rproc_stop(struct rproc *rproc) } static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da, - int len, u64 *sys) + size_t len, u64 *sys) { const struct imx_rproc_dcfg *dcfg = priv->dcfg; int i; @@ -203,19 +203,19 @@ static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da, } } - dev_warn(priv->dev, "Translation failed: da = 0x%llx len = 0x%x\n", + dev_warn(priv->dev, "Translation failed: da = 0x%llx len = 0x%zx\n", da, len); return -ENOENT; } -static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, int len) +static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct imx_rproc *priv = rproc->priv; void *va = NULL; u64 sys; int i; - if (len <= 0) + if (len == 0) return NULL; /* @@ -235,7 +235,8 @@ static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, int len) } } - dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%x va = 0x%p\n", da, len, va); + dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%zx va = 0x%p\n", + da, len, va); return va; } diff --git a/drivers/remoteproc/keystone_remoteproc.c b/drivers/remoteproc/keystone_remoteproc.c index 5c4658f00b3d..cd266163a65f 100644 --- a/drivers/remoteproc/keystone_remoteproc.c +++ b/drivers/remoteproc/keystone_remoteproc.c @@ -246,7 +246,7 @@ static void keystone_rproc_kick(struct rproc *rproc, int vqid) * can be used either by the remoteproc core for loading (when using kernel * remoteproc loader), or by any rpmsg bus drivers. */ -static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len) +static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct keystone_rproc *ksproc = rproc->priv; void __iomem *va = NULL; @@ -255,7 +255,7 @@ static void *keystone_rproc_da_to_va(struct rproc *rproc, u64 da, int len) size_t size; int i; - if (len <= 0) + if (len == 0) return NULL; for (i = 0; i < ksproc->num_mems; i++) { diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index 7ccdf64ff3ea..ea3743e7e794 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -320,7 +320,7 @@ static int scp_start(struct rproc *rproc) return ret; } -static void *scp_da_to_va(struct rproc *rproc, u64 da, int len) +static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; int offset; diff --git a/drivers/remoteproc/omap_remoteproc.c b/drivers/remoteproc/omap_remoteproc.c index 6398194075aa..cdb546f7232e 100644 --- a/drivers/remoteproc/omap_remoteproc.c +++ b/drivers/remoteproc/omap_remoteproc.c @@ -2,7 +2,7 @@ /* * OMAP Remote Processor driver * - * Copyright (C) 2011 Texas Instruments, Inc. + * Copyright (C) 2011-2020 Texas Instruments Incorporated - http://www.ti.com/ * Copyright (C) 2011 Google, Inc. * * Ohad Ben-Cohen @@ -15,30 +15,465 @@ #include #include +#include +#include #include +#include +#include +#include #include +#include #include +#include #include #include +#include #include +#include +#include +#include +#include -#include +#include #include "omap_remoteproc.h" #include "remoteproc_internal.h" +/* default auto-suspend delay (ms) */ +#define DEFAULT_AUTOSUSPEND_DELAY 10000 + +/** + * struct omap_rproc_boot_data - boot data structure for the DSP omap rprocs + * @syscon: regmap handle for the system control configuration module + * @boot_reg: boot register offset within the @syscon regmap + * @boot_reg_shift: bit-field shift required for the boot address value in + * @boot_reg + */ +struct omap_rproc_boot_data { + struct regmap *syscon; + unsigned int boot_reg; + unsigned int boot_reg_shift; +}; + +/** + * struct omap_rproc_mem - internal memory structure + * @cpu_addr: MPU virtual address of the memory region + * @bus_addr: bus address used to access the memory region + * @dev_addr: device address of the memory region from DSP view + * @size: size of the memory region + */ +struct omap_rproc_mem { + void __iomem *cpu_addr; + phys_addr_t bus_addr; + u32 dev_addr; + size_t size; +}; + +/** + * struct omap_rproc_timer - data structure for a timer used by a omap rproc + * @odt: timer pointer + * @timer_ops: OMAP dmtimer ops for @odt timer + * @irq: timer irq + */ +struct omap_rproc_timer { + struct omap_dm_timer *odt; + const struct omap_dm_timer_ops *timer_ops; + int irq; +}; + /** * struct omap_rproc - omap remote processor state * @mbox: mailbox channel handle * @client: mailbox client to request the mailbox channel + * @boot_data: boot data structure for setting processor boot address + * @mem: internal memory regions data + * @num_mems: number of internal memory regions + * @num_timers: number of rproc timer(s) + * @num_wd_timers: number of rproc watchdog timers + * @timers: timer(s) info used by rproc + * @autosuspend_delay: auto-suspend delay value to be used for runtime pm + * @need_resume: if true a resume is needed in the system resume callback * @rproc: rproc handle + * @reset: reset handle + * @pm_comp: completion primitive to sync for suspend response + * @fck: functional clock for the remoteproc + * @suspend_acked: state machine flag to store the suspend request ack */ struct omap_rproc { struct mbox_chan *mbox; struct mbox_client client; + struct omap_rproc_boot_data *boot_data; + struct omap_rproc_mem *mem; + int num_mems; + int num_timers; + int num_wd_timers; + struct omap_rproc_timer *timers; + int autosuspend_delay; + bool need_resume; struct rproc *rproc; + struct reset_control *reset; + struct completion pm_comp; + struct clk *fck; + bool suspend_acked; }; +/** + * struct omap_rproc_mem_data - memory definitions for an omap remote processor + * @name: name for this memory entry + * @dev_addr: device address for the memory entry + */ +struct omap_rproc_mem_data { + const char *name; + const u32 dev_addr; +}; + +/** + * struct omap_rproc_dev_data - device data for the omap remote processor + * @device_name: device name of the remote processor + * @mems: memory definitions for this remote processor + */ +struct omap_rproc_dev_data { + const char *device_name; + const struct omap_rproc_mem_data *mems; +}; + +/** + * omap_rproc_request_timer() - request a timer for a remoteproc + * @dev: device requesting the timer + * @np: device node pointer to the desired timer + * @timer: handle to a struct omap_rproc_timer to return the timer handle + * + * This helper function is used primarily to request a timer associated with + * a remoteproc. The returned handle is stored in the .odt field of the + * @timer structure passed in, and is used to invoke other timer specific + * ops (like starting a timer either during device initialization or during + * a resume operation, or for stopping/freeing a timer). + * + * Return: 0 on success, otherwise an appropriate failure + */ +static int omap_rproc_request_timer(struct device *dev, struct device_node *np, + struct omap_rproc_timer *timer) +{ + int ret; + + timer->odt = timer->timer_ops->request_by_node(np); + if (!timer->odt) { + dev_err(dev, "request for timer node %p failed\n", np); + return -EBUSY; + } + + ret = timer->timer_ops->set_source(timer->odt, OMAP_TIMER_SRC_SYS_CLK); + if (ret) { + dev_err(dev, "error setting OMAP_TIMER_SRC_SYS_CLK as source for timer node %p\n", + np); + timer->timer_ops->free(timer->odt); + return ret; + } + + /* clean counter, remoteproc code will set the value */ + timer->timer_ops->set_load(timer->odt, 0, 0); + + return 0; +} + +/** + * omap_rproc_start_timer() - start a timer for a remoteproc + * @timer: handle to a OMAP rproc timer + * + * This helper function is used to start a timer associated with a remoteproc, + * obtained using the request_timer ops. The helper function needs to be + * invoked by the driver to start the timer (during device initialization) + * or to just resume the timer. + * + * Return: 0 on success, otherwise a failure as returned by DMTimer ops + */ +static inline int omap_rproc_start_timer(struct omap_rproc_timer *timer) +{ + return timer->timer_ops->start(timer->odt); +} + +/** + * omap_rproc_stop_timer() - stop a timer for a remoteproc + * @timer: handle to a OMAP rproc timer + * + * This helper function is used to disable a timer associated with a + * remoteproc, and needs to be called either during a device shutdown + * or suspend operation. The separate helper function allows the driver + * to just stop a timer without having to release the timer during a + * suspend operation. + * + * Return: 0 on success, otherwise a failure as returned by DMTimer ops + */ +static inline int omap_rproc_stop_timer(struct omap_rproc_timer *timer) +{ + return timer->timer_ops->stop(timer->odt); +} + +/** + * omap_rproc_release_timer() - release a timer for a remoteproc + * @timer: handle to a OMAP rproc timer + * + * This helper function is used primarily to release a timer associated + * with a remoteproc. The dmtimer will be available for other clients to + * use once released. + * + * Return: 0 on success, otherwise a failure as returned by DMTimer ops + */ +static inline int omap_rproc_release_timer(struct omap_rproc_timer *timer) +{ + return timer->timer_ops->free(timer->odt); +} + +/** + * omap_rproc_get_timer_irq() - get the irq for a timer + * @timer: handle to a OMAP rproc timer + * + * This function is used to get the irq associated with a watchdog timer. The + * function is called by the OMAP remoteproc driver to register a interrupt + * handler to handle watchdog events on the remote processor. + * + * Return: irq id on success, otherwise a failure as returned by DMTimer ops + */ +static inline int omap_rproc_get_timer_irq(struct omap_rproc_timer *timer) +{ + return timer->timer_ops->get_irq(timer->odt); +} + +/** + * omap_rproc_ack_timer_irq() - acknowledge a timer irq + * @timer: handle to a OMAP rproc timer + * + * This function is used to clear the irq associated with a watchdog timer. The + * The function is called by the OMAP remoteproc upon a watchdog event on the + * remote processor to clear the interrupt status of the watchdog timer. + */ +static inline void omap_rproc_ack_timer_irq(struct omap_rproc_timer *timer) +{ + timer->timer_ops->write_status(timer->odt, OMAP_TIMER_INT_OVERFLOW); +} + +/** + * omap_rproc_watchdog_isr() - Watchdog ISR handler for remoteproc device + * @irq: IRQ number associated with a watchdog timer + * @data: IRQ handler data + * + * This ISR routine executes the required necessary low-level code to + * acknowledge a watchdog timer interrupt. There can be multiple watchdog + * timers associated with a rproc (like IPUs which have 2 watchdog timers, + * one per Cortex M3/M4 core), so a lookup has to be performed to identify + * the timer to acknowledge its interrupt. + * + * The function also invokes rproc_report_crash to report the watchdog event + * to the remoteproc driver core, to trigger a recovery. + * + * Return: IRQ_HANDLED on success, otherwise IRQ_NONE + */ +static irqreturn_t omap_rproc_watchdog_isr(int irq, void *data) +{ + struct rproc *rproc = data; + struct omap_rproc *oproc = rproc->priv; + struct device *dev = rproc->dev.parent; + struct omap_rproc_timer *timers = oproc->timers; + struct omap_rproc_timer *wd_timer = NULL; + int num_timers = oproc->num_timers + oproc->num_wd_timers; + int i; + + for (i = oproc->num_timers; i < num_timers; i++) { + if (timers[i].irq > 0 && irq == timers[i].irq) { + wd_timer = &timers[i]; + break; + } + } + + if (!wd_timer) { + dev_err(dev, "invalid timer\n"); + return IRQ_NONE; + } + + omap_rproc_ack_timer_irq(wd_timer); + + rproc_report_crash(rproc, RPROC_WATCHDOG); + + return IRQ_HANDLED; +} + +/** + * omap_rproc_enable_timers() - enable the timers for a remoteproc + * @rproc: handle of a remote processor + * @configure: boolean flag used to acquire and configure the timer handle + * + * This function is used primarily to enable the timers associated with + * a remoteproc. The configure flag is provided to allow the driver to + * to either acquire and start a timer (during device initialization) or + * to just start a timer (during a resume operation). + * + * Return: 0 on success, otherwise an appropriate failure + */ +static int omap_rproc_enable_timers(struct rproc *rproc, bool configure) +{ + int i; + int ret = 0; + struct platform_device *tpdev; + struct dmtimer_platform_data *tpdata; + const struct omap_dm_timer_ops *timer_ops; + struct omap_rproc *oproc = rproc->priv; + struct omap_rproc_timer *timers = oproc->timers; + struct device *dev = rproc->dev.parent; + struct device_node *np = NULL; + int num_timers = oproc->num_timers + oproc->num_wd_timers; + + if (!num_timers) + return 0; + + if (!configure) + goto start_timers; + + for (i = 0; i < num_timers; i++) { + if (i < oproc->num_timers) + np = of_parse_phandle(dev->of_node, "ti,timers", i); + else + np = of_parse_phandle(dev->of_node, + "ti,watchdog-timers", + (i - oproc->num_timers)); + if (!np) { + ret = -ENXIO; + dev_err(dev, "device node lookup for timer at index %d failed: %d\n", + i < oproc->num_timers ? i : + i - oproc->num_timers, ret); + goto free_timers; + } + + tpdev = of_find_device_by_node(np); + if (!tpdev) { + ret = -ENODEV; + dev_err(dev, "could not get timer platform device\n"); + goto put_node; + } + + tpdata = dev_get_platdata(&tpdev->dev); + put_device(&tpdev->dev); + if (!tpdata) { + ret = -EINVAL; + dev_err(dev, "dmtimer pdata structure NULL\n"); + goto put_node; + } + + timer_ops = tpdata->timer_ops; + if (!timer_ops || !timer_ops->request_by_node || + !timer_ops->set_source || !timer_ops->set_load || + !timer_ops->free || !timer_ops->start || + !timer_ops->stop || !timer_ops->get_irq || + !timer_ops->write_status) { + ret = -EINVAL; + dev_err(dev, "device does not have required timer ops\n"); + goto put_node; + } + + timers[i].irq = -1; + timers[i].timer_ops = timer_ops; + ret = omap_rproc_request_timer(dev, np, &timers[i]); + if (ret) { + dev_err(dev, "request for timer %p failed: %d\n", np, + ret); + goto put_node; + } + of_node_put(np); + + if (i >= oproc->num_timers) { + timers[i].irq = omap_rproc_get_timer_irq(&timers[i]); + if (timers[i].irq < 0) { + dev_err(dev, "get_irq for timer %p failed: %d\n", + np, timers[i].irq); + ret = -EBUSY; + goto free_timers; + } + + ret = request_irq(timers[i].irq, + omap_rproc_watchdog_isr, IRQF_SHARED, + "rproc-wdt", rproc); + if (ret) { + dev_err(dev, "error requesting irq for timer %p\n", + np); + omap_rproc_release_timer(&timers[i]); + timers[i].odt = NULL; + timers[i].timer_ops = NULL; + timers[i].irq = -1; + goto free_timers; + } + } + } + +start_timers: + for (i = 0; i < num_timers; i++) { + ret = omap_rproc_start_timer(&timers[i]); + if (ret) { + dev_err(dev, "start timer %p failed failed: %d\n", np, + ret); + break; + } + } + if (ret) { + while (i >= 0) { + omap_rproc_stop_timer(&timers[i]); + i--; + } + goto put_node; + } + return 0; + +put_node: + if (configure) + of_node_put(np); +free_timers: + while (i--) { + if (i >= oproc->num_timers) + free_irq(timers[i].irq, rproc); + omap_rproc_release_timer(&timers[i]); + timers[i].odt = NULL; + timers[i].timer_ops = NULL; + timers[i].irq = -1; + } + + return ret; +} + +/** + * omap_rproc_disable_timers() - disable the timers for a remoteproc + * @rproc: handle of a remote processor + * @configure: boolean flag used to release the timer handle + * + * This function is used primarily to disable the timers associated with + * a remoteproc. The configure flag is provided to allow the driver to + * to either stop and release a timer (during device shutdown) or to just + * stop a timer (during a suspend operation). + * + * Return: 0 on success or no timers + */ +static int omap_rproc_disable_timers(struct rproc *rproc, bool configure) +{ + int i; + struct omap_rproc *oproc = rproc->priv; + struct omap_rproc_timer *timers = oproc->timers; + int num_timers = oproc->num_timers + oproc->num_wd_timers; + + if (!num_timers) + return 0; + + for (i = 0; i < num_timers; i++) { + omap_rproc_stop_timer(&timers[i]); + if (configure) { + if (i >= oproc->num_timers) + free_irq(timers[i].irq, rproc); + omap_rproc_release_timer(&timers[i]); + timers[i].odt = NULL; + timers[i].timer_ops = NULL; + timers[i].irq = -1; + } + } + + return 0; +} + /** * omap_rproc_mbox_callback() - inbound mailbox message handler * @client: mailbox client pointer used for requesting the mailbox channel @@ -65,13 +500,29 @@ static void omap_rproc_mbox_callback(struct mbox_client *client, void *data) switch (msg) { case RP_MBOX_CRASH: - /* just log this for now. later, we'll also do recovery */ + /* + * remoteproc detected an exception, notify the rproc core. + * The remoteproc core will handle the recovery. + */ dev_err(dev, "omap rproc %s crashed\n", name); + rproc_report_crash(oproc->rproc, RPROC_FATAL_ERROR); break; case RP_MBOX_ECHO_REPLY: dev_info(dev, "received echo reply from %s\n", name); break; + case RP_MBOX_SUSPEND_ACK: + /* Fall through */ + case RP_MBOX_SUSPEND_CANCEL: + oproc->suspend_acked = msg == RP_MBOX_SUSPEND_ACK; + complete(&oproc->pm_comp); + break; default: + if (msg >= RP_MBOX_READY && msg < RP_MBOX_END_MSG) + return; + if (msg > oproc->rproc->max_notifyid) { + dev_dbg(dev, "dropping unknown message 0x%x", msg); + return; + } /* msg contains the index of the triggered vring */ if (rproc_vq_interrupt(oproc->rproc, msg) == IRQ_NONE) dev_dbg(dev, "no message was found in vqid %d\n", msg); @@ -85,11 +536,52 @@ static void omap_rproc_kick(struct rproc *rproc, int vqid) struct device *dev = rproc->dev.parent; int ret; + /* wake up the rproc before kicking it */ + ret = pm_runtime_get_sync(dev); + if (WARN_ON(ret < 0)) { + dev_err(dev, "pm_runtime_get_sync() failed during kick, ret = %d\n", + ret); + pm_runtime_put_noidle(dev); + return; + } + /* send the index of the triggered virtqueue in the mailbox payload */ ret = mbox_send_message(oproc->mbox, (void *)vqid); if (ret < 0) dev_err(dev, "failed to send mailbox message, status = %d\n", ret); + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); +} + +/** + * omap_rproc_write_dsp_boot_addr() - set boot address for DSP remote processor + * @rproc: handle of a remote processor + * + * Set boot address for a supported DSP remote processor. + * + * Return: 0 on success, or -EINVAL if boot address is not aligned properly + */ +static int omap_rproc_write_dsp_boot_addr(struct rproc *rproc) +{ + struct device *dev = rproc->dev.parent; + struct omap_rproc *oproc = rproc->priv; + struct omap_rproc_boot_data *bdata = oproc->boot_data; + u32 offset = bdata->boot_reg; + u32 value; + u32 mask; + + if (rproc->bootaddr & (SZ_1K - 1)) { + dev_err(dev, "invalid boot address 0x%llx, must be aligned on a 1KB boundary\n", + rproc->bootaddr); + return -EINVAL; + } + + value = rproc->bootaddr >> bdata->boot_reg_shift; + mask = ~(SZ_1K - 1) >> bdata->boot_reg_shift; + + return regmap_update_bits(bdata->syscon, offset, mask, value); } /* @@ -103,13 +595,14 @@ static int omap_rproc_start(struct rproc *rproc) { struct omap_rproc *oproc = rproc->priv; struct device *dev = rproc->dev.parent; - struct platform_device *pdev = to_platform_device(dev); - struct omap_rproc_pdata *pdata = pdev->dev.platform_data; int ret; struct mbox_client *client = &oproc->client; - if (pdata->set_bootaddr) - pdata->set_bootaddr(rproc->bootaddr); + if (oproc->boot_data) { + ret = omap_rproc_write_dsp_boot_addr(rproc); + if (ret) + return ret; + } client->dev = dev; client->tx_done = NULL; @@ -117,7 +610,7 @@ static int omap_rproc_start(struct rproc *rproc) client->tx_block = false; client->knows_txdone = false; - oproc->mbox = omap_mbox_request_channel(client, pdata->mbox_name); + oproc->mbox = mbox_request_channel(client, 0); if (IS_ERR(oproc->mbox)) { ret = -EBUSY; dev_err(dev, "mbox_request_channel failed: %ld\n", @@ -138,14 +631,34 @@ static int omap_rproc_start(struct rproc *rproc) goto put_mbox; } - ret = pdata->device_enable(pdev); + ret = omap_rproc_enable_timers(rproc, true); if (ret) { - dev_err(dev, "omap_device_enable failed: %d\n", ret); + dev_err(dev, "omap_rproc_enable_timers failed: %d\n", ret); goto put_mbox; } + ret = reset_control_deassert(oproc->reset); + if (ret) { + dev_err(dev, "reset control deassert failed: %d\n", ret); + goto disable_timers; + } + + /* + * remote processor is up, so update the runtime pm status and + * enable the auto-suspend. The device usage count is incremented + * manually for balancing it for auto-suspend + */ + pm_runtime_set_active(dev); + pm_runtime_use_autosuspend(dev); + pm_runtime_get_noresume(dev); + pm_runtime_enable(dev); + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + return 0; +disable_timers: + omap_rproc_disable_timers(rproc, true); put_mbox: mbox_free_channel(oproc->mbox); return ret; @@ -155,32 +668,638 @@ static int omap_rproc_start(struct rproc *rproc) static int omap_rproc_stop(struct rproc *rproc) { struct device *dev = rproc->dev.parent; - struct platform_device *pdev = to_platform_device(dev); - struct omap_rproc_pdata *pdata = pdev->dev.platform_data; struct omap_rproc *oproc = rproc->priv; int ret; - ret = pdata->device_shutdown(pdev); - if (ret) + /* + * cancel any possible scheduled runtime suspend by incrementing + * the device usage count, and resuming the device. The remoteproc + * also needs to be woken up if suspended, to avoid the remoteproc + * OS to continue to remember any context that it has saved, and + * avoid potential issues in misindentifying a subsequent device + * reboot as a power restore boot + */ + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + pm_runtime_put_noidle(dev); return ret; + } + + ret = reset_control_assert(oproc->reset); + if (ret) + goto out; + + ret = omap_rproc_disable_timers(rproc, true); + if (ret) + goto enable_device; mbox_free_channel(oproc->mbox); + /* + * update the runtime pm states and status now that the remoteproc + * has stopped + */ + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + pm_runtime_put_noidle(dev); + pm_runtime_set_suspended(dev); + return 0; + +enable_device: + reset_control_deassert(oproc->reset); +out: + /* schedule the next auto-suspend */ + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + return ret; +} + +/** + * omap_rproc_da_to_va() - internal memory translation helper + * @rproc: remote processor to apply the address translation for + * @da: device address to translate + * @len: length of the memory buffer + * + * Custom function implementing the rproc .da_to_va ops to provide address + * translation (device address to kernel virtual address) for internal RAMs + * present in a DSP or IPU device). The translated addresses can be used + * either by the remoteproc core for loading, or by any rpmsg bus drivers. + * + * Return: translated virtual address in kernel memory space on success, + * or NULL on failure. + */ +static void *omap_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len) +{ + struct omap_rproc *oproc = rproc->priv; + int i; + u32 offset; + + if (len <= 0) + return NULL; + + if (!oproc->num_mems) + return NULL; + + for (i = 0; i < oproc->num_mems; i++) { + if (da >= oproc->mem[i].dev_addr && da + len <= + oproc->mem[i].dev_addr + oproc->mem[i].size) { + offset = da - oproc->mem[i].dev_addr; + /* __force to make sparse happy with type conversion */ + return (__force void *)(oproc->mem[i].cpu_addr + + offset); + } + } + + return NULL; } static const struct rproc_ops omap_rproc_ops = { .start = omap_rproc_start, .stop = omap_rproc_stop, .kick = omap_rproc_kick, + .da_to_va = omap_rproc_da_to_va, }; +#ifdef CONFIG_PM +static bool _is_rproc_in_standby(struct omap_rproc *oproc) +{ + return ti_clk_is_in_standby(oproc->fck); +} + +/* 1 sec is long enough time to let the remoteproc side suspend the device */ +#define DEF_SUSPEND_TIMEOUT 1000 +static int _omap_rproc_suspend(struct rproc *rproc, bool auto_suspend) +{ + struct device *dev = rproc->dev.parent; + struct omap_rproc *oproc = rproc->priv; + unsigned long to = msecs_to_jiffies(DEF_SUSPEND_TIMEOUT); + unsigned long ta = jiffies + to; + u32 suspend_msg = auto_suspend ? + RP_MBOX_SUSPEND_AUTO : RP_MBOX_SUSPEND_SYSTEM; + int ret; + + reinit_completion(&oproc->pm_comp); + oproc->suspend_acked = false; + ret = mbox_send_message(oproc->mbox, (void *)suspend_msg); + if (ret < 0) { + dev_err(dev, "PM mbox_send_message failed: %d\n", ret); + return ret; + } + + ret = wait_for_completion_timeout(&oproc->pm_comp, to); + if (!oproc->suspend_acked) + return -EBUSY; + + /* + * The remoteproc side is returning the ACK message before saving the + * context, because the context saving is performed within a SYS/BIOS + * function, and it cannot have any inter-dependencies against the IPC + * layer. Also, as the SYS/BIOS needs to preserve properly the processor + * register set, sending this ACK or signalling the completion of the + * context save through a shared memory variable can never be the + * absolute last thing to be executed on the remoteproc side, and the + * MPU cannot use the ACK message as a sync point to put the remoteproc + * into reset. The only way to ensure that the remote processor has + * completed saving the context is to check that the module has reached + * STANDBY state (after saving the context, the SYS/BIOS executes the + * appropriate target-specific WFI instruction causing the module to + * enter STANDBY). + */ + while (!_is_rproc_in_standby(oproc)) { + if (time_after(jiffies, ta)) + return -ETIME; + schedule(); + } + + ret = reset_control_assert(oproc->reset); + if (ret) { + dev_err(dev, "reset assert during suspend failed %d\n", ret); + return ret; + } + + ret = omap_rproc_disable_timers(rproc, false); + if (ret) { + dev_err(dev, "disabling timers during suspend failed %d\n", + ret); + goto enable_device; + } + + /* + * IOMMUs would have to be disabled specifically for runtime suspend. + * They are handled automatically through System PM callbacks for + * regular system suspend + */ + if (auto_suspend) { + ret = omap_iommu_domain_deactivate(rproc->domain); + if (ret) { + dev_err(dev, "iommu domain deactivate failed %d\n", + ret); + goto enable_timers; + } + } + + return 0; + +enable_timers: + /* ignore errors on re-enabling code */ + omap_rproc_enable_timers(rproc, false); +enable_device: + reset_control_deassert(oproc->reset); + return ret; +} + +static int _omap_rproc_resume(struct rproc *rproc, bool auto_suspend) +{ + struct device *dev = rproc->dev.parent; + struct omap_rproc *oproc = rproc->priv; + int ret; + + /* + * IOMMUs would have to be enabled specifically for runtime resume. + * They would have been already enabled automatically through System + * PM callbacks for regular system resume + */ + if (auto_suspend) { + ret = omap_iommu_domain_activate(rproc->domain); + if (ret) { + dev_err(dev, "omap_iommu activate failed %d\n", ret); + goto out; + } + } + + /* boot address could be lost after suspend, so restore it */ + if (oproc->boot_data) { + ret = omap_rproc_write_dsp_boot_addr(rproc); + if (ret) { + dev_err(dev, "boot address restore failed %d\n", ret); + goto suspend_iommu; + } + } + + ret = omap_rproc_enable_timers(rproc, false); + if (ret) { + dev_err(dev, "enabling timers during resume failed %d\n", ret); + goto suspend_iommu; + } + + ret = reset_control_deassert(oproc->reset); + if (ret) { + dev_err(dev, "reset deassert during resume failed %d\n", ret); + goto disable_timers; + } + + return 0; + +disable_timers: + omap_rproc_disable_timers(rproc, false); +suspend_iommu: + if (auto_suspend) + omap_iommu_domain_deactivate(rproc->domain); +out: + return ret; +} + +static int __maybe_unused omap_rproc_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct rproc *rproc = platform_get_drvdata(pdev); + struct omap_rproc *oproc = rproc->priv; + int ret = 0; + + mutex_lock(&rproc->lock); + if (rproc->state == RPROC_OFFLINE) + goto out; + + if (rproc->state == RPROC_SUSPENDED) + goto out; + + if (rproc->state != RPROC_RUNNING) { + ret = -EBUSY; + goto out; + } + + ret = _omap_rproc_suspend(rproc, false); + if (ret) { + dev_err(dev, "suspend failed %d\n", ret); + goto out; + } + + /* + * remoteproc is running at the time of system suspend, so remember + * it so as to wake it up during system resume + */ + oproc->need_resume = true; + rproc->state = RPROC_SUSPENDED; + +out: + mutex_unlock(&rproc->lock); + return ret; +} + +static int __maybe_unused omap_rproc_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct rproc *rproc = platform_get_drvdata(pdev); + struct omap_rproc *oproc = rproc->priv; + int ret = 0; + + mutex_lock(&rproc->lock); + if (rproc->state == RPROC_OFFLINE) + goto out; + + if (rproc->state != RPROC_SUSPENDED) { + ret = -EBUSY; + goto out; + } + + /* + * remoteproc was auto-suspended at the time of system suspend, + * so no need to wake-up the processor (leave it in suspended + * state, will be woken up during a subsequent runtime_resume) + */ + if (!oproc->need_resume) + goto out; + + ret = _omap_rproc_resume(rproc, false); + if (ret) { + dev_err(dev, "resume failed %d\n", ret); + goto out; + } + + oproc->need_resume = false; + rproc->state = RPROC_RUNNING; + + pm_runtime_mark_last_busy(dev); +out: + mutex_unlock(&rproc->lock); + return ret; +} + +static int omap_rproc_runtime_suspend(struct device *dev) +{ + struct rproc *rproc = dev_get_drvdata(dev); + struct omap_rproc *oproc = rproc->priv; + int ret; + + mutex_lock(&rproc->lock); + if (rproc->state == RPROC_CRASHED) { + dev_dbg(dev, "rproc cannot be runtime suspended when crashed!\n"); + ret = -EBUSY; + goto out; + } + + if (WARN_ON(rproc->state != RPROC_RUNNING)) { + dev_err(dev, "rproc cannot be runtime suspended when not running!\n"); + ret = -EBUSY; + goto out; + } + + /* + * do not even attempt suspend if the remote processor is not + * idled for runtime auto-suspend + */ + if (!_is_rproc_in_standby(oproc)) { + ret = -EBUSY; + goto abort; + } + + ret = _omap_rproc_suspend(rproc, true); + if (ret) + goto abort; + + rproc->state = RPROC_SUSPENDED; + mutex_unlock(&rproc->lock); + return 0; + +abort: + pm_runtime_mark_last_busy(dev); +out: + mutex_unlock(&rproc->lock); + return ret; +} + +static int omap_rproc_runtime_resume(struct device *dev) +{ + struct rproc *rproc = dev_get_drvdata(dev); + int ret; + + mutex_lock(&rproc->lock); + if (WARN_ON(rproc->state != RPROC_SUSPENDED)) { + dev_err(dev, "rproc cannot be runtime resumed if not suspended! state=%d\n", + rproc->state); + ret = -EBUSY; + goto out; + } + + ret = _omap_rproc_resume(rproc, true); + if (ret) { + dev_err(dev, "runtime resume failed %d\n", ret); + goto out; + } + + rproc->state = RPROC_RUNNING; +out: + mutex_unlock(&rproc->lock); + return ret; +} +#endif /* CONFIG_PM */ + +static const struct omap_rproc_mem_data ipu_mems[] = { + { .name = "l2ram", .dev_addr = 0x20000000 }, + { }, +}; + +static const struct omap_rproc_mem_data dra7_dsp_mems[] = { + { .name = "l2ram", .dev_addr = 0x800000 }, + { .name = "l1pram", .dev_addr = 0xe00000 }, + { .name = "l1dram", .dev_addr = 0xf00000 }, + { }, +}; + +static const struct omap_rproc_dev_data omap4_dsp_dev_data = { + .device_name = "dsp", +}; + +static const struct omap_rproc_dev_data omap4_ipu_dev_data = { + .device_name = "ipu", + .mems = ipu_mems, +}; + +static const struct omap_rproc_dev_data omap5_dsp_dev_data = { + .device_name = "dsp", +}; + +static const struct omap_rproc_dev_data omap5_ipu_dev_data = { + .device_name = "ipu", + .mems = ipu_mems, +}; + +static const struct omap_rproc_dev_data dra7_dsp_dev_data = { + .device_name = "dsp", + .mems = dra7_dsp_mems, +}; + +static const struct omap_rproc_dev_data dra7_ipu_dev_data = { + .device_name = "ipu", + .mems = ipu_mems, +}; + +static const struct of_device_id omap_rproc_of_match[] = { + { + .compatible = "ti,omap4-dsp", + .data = &omap4_dsp_dev_data, + }, + { + .compatible = "ti,omap4-ipu", + .data = &omap4_ipu_dev_data, + }, + { + .compatible = "ti,omap5-dsp", + .data = &omap5_dsp_dev_data, + }, + { + .compatible = "ti,omap5-ipu", + .data = &omap5_ipu_dev_data, + }, + { + .compatible = "ti,dra7-dsp", + .data = &dra7_dsp_dev_data, + }, + { + .compatible = "ti,dra7-ipu", + .data = &dra7_ipu_dev_data, + }, + { + /* end */ + }, +}; +MODULE_DEVICE_TABLE(of, omap_rproc_of_match); + +static const char *omap_rproc_get_firmware(struct platform_device *pdev) +{ + const char *fw_name; + int ret; + + ret = of_property_read_string(pdev->dev.of_node, "firmware-name", + &fw_name); + if (ret) + return ERR_PTR(ret); + + return fw_name; +} + +static int omap_rproc_get_boot_data(struct platform_device *pdev, + struct rproc *rproc) +{ + struct device_node *np = pdev->dev.of_node; + struct omap_rproc *oproc = rproc->priv; + const struct omap_rproc_dev_data *data; + int ret; + + data = of_device_get_match_data(&pdev->dev); + if (!data) + return -ENODEV; + + if (!of_property_read_bool(np, "ti,bootreg")) + return 0; + + oproc->boot_data = devm_kzalloc(&pdev->dev, sizeof(*oproc->boot_data), + GFP_KERNEL); + if (!oproc->boot_data) + return -ENOMEM; + + oproc->boot_data->syscon = + syscon_regmap_lookup_by_phandle(np, "ti,bootreg"); + if (IS_ERR(oproc->boot_data->syscon)) { + ret = PTR_ERR(oproc->boot_data->syscon); + return ret; + } + + if (of_property_read_u32_index(np, "ti,bootreg", 1, + &oproc->boot_data->boot_reg)) { + dev_err(&pdev->dev, "couldn't get the boot register\n"); + return -EINVAL; + } + + of_property_read_u32_index(np, "ti,bootreg", 2, + &oproc->boot_data->boot_reg_shift); + + return 0; +} + +static int omap_rproc_of_get_internal_memories(struct platform_device *pdev, + struct rproc *rproc) +{ + struct omap_rproc *oproc = rproc->priv; + struct device *dev = &pdev->dev; + const struct omap_rproc_dev_data *data; + struct resource *res; + int num_mems; + int i; + + data = of_device_get_match_data(dev); + if (!data) + return -ENODEV; + + if (!data->mems) + return 0; + + num_mems = of_property_count_elems_of_size(dev->of_node, "reg", + sizeof(u32)) / 2; + + oproc->mem = devm_kcalloc(dev, num_mems, sizeof(*oproc->mem), + GFP_KERNEL); + if (!oproc->mem) + return -ENOMEM; + + for (i = 0; data->mems[i].name; i++) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + data->mems[i].name); + if (!res) { + dev_err(dev, "no memory defined for %s\n", + data->mems[i].name); + return -ENOMEM; + } + oproc->mem[i].cpu_addr = devm_ioremap_resource(dev, res); + if (IS_ERR(oproc->mem[i].cpu_addr)) { + dev_err(dev, "failed to parse and map %s memory\n", + data->mems[i].name); + return PTR_ERR(oproc->mem[i].cpu_addr); + } + oproc->mem[i].bus_addr = res->start; + oproc->mem[i].dev_addr = data->mems[i].dev_addr; + oproc->mem[i].size = resource_size(res); + + dev_dbg(dev, "memory %8s: bus addr %pa size 0x%x va %pK da 0x%x\n", + data->mems[i].name, &oproc->mem[i].bus_addr, + oproc->mem[i].size, oproc->mem[i].cpu_addr, + oproc->mem[i].dev_addr); + } + oproc->num_mems = num_mems; + + return 0; +} + +#ifdef CONFIG_OMAP_REMOTEPROC_WATCHDOG +static int omap_rproc_count_wdog_timers(struct device *dev) +{ + struct device_node *np = dev->of_node; + int ret; + + ret = of_count_phandle_with_args(np, "ti,watchdog-timers", NULL); + if (ret <= 0) { + dev_dbg(dev, "device does not have watchdog timers, status = %d\n", + ret); + ret = 0; + } + + return ret; +} +#else +static int omap_rproc_count_wdog_timers(struct device *dev) +{ + return 0; +} +#endif + +static int omap_rproc_of_get_timers(struct platform_device *pdev, + struct rproc *rproc) +{ + struct device_node *np = pdev->dev.of_node; + struct omap_rproc *oproc = rproc->priv; + struct device *dev = &pdev->dev; + int num_timers; + + /* + * Timer nodes are directly used in client nodes as phandles, so + * retrieve the count using appropriate size + */ + oproc->num_timers = of_count_phandle_with_args(np, "ti,timers", NULL); + if (oproc->num_timers <= 0) { + dev_dbg(dev, "device does not have timers, status = %d\n", + oproc->num_timers); + oproc->num_timers = 0; + } + + oproc->num_wd_timers = omap_rproc_count_wdog_timers(dev); + + num_timers = oproc->num_timers + oproc->num_wd_timers; + if (num_timers) { + oproc->timers = devm_kcalloc(dev, num_timers, + sizeof(*oproc->timers), + GFP_KERNEL); + if (!oproc->timers) + return -ENOMEM; + + dev_dbg(dev, "device has %d tick timers and %d watchdog timers\n", + oproc->num_timers, oproc->num_wd_timers); + } + + return 0; +} + static int omap_rproc_probe(struct platform_device *pdev) { - struct omap_rproc_pdata *pdata = pdev->dev.platform_data; + struct device_node *np = pdev->dev.of_node; struct omap_rproc *oproc; struct rproc *rproc; + const char *firmware; int ret; + struct reset_control *reset; + + if (!np) { + dev_err(&pdev->dev, "only DT-based devices are supported\n"); + return -ENODEV; + } + + reset = devm_reset_control_array_get_exclusive(&pdev->dev); + if (IS_ERR(reset)) + return PTR_ERR(reset); + + firmware = omap_rproc_get_firmware(pdev); + if (IS_ERR(firmware)) + return PTR_ERR(firmware); ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { @@ -188,24 +1307,60 @@ static int omap_rproc_probe(struct platform_device *pdev) return ret; } - rproc = rproc_alloc(&pdev->dev, pdata->name, &omap_rproc_ops, - pdata->firmware, sizeof(*oproc)); + rproc = rproc_alloc(&pdev->dev, dev_name(&pdev->dev), &omap_rproc_ops, + firmware, sizeof(*oproc)); if (!rproc) return -ENOMEM; oproc = rproc->priv; oproc->rproc = rproc; + oproc->reset = reset; /* All existing OMAP IPU and DSP processors have an MMU */ rproc->has_iommu = true; + ret = omap_rproc_of_get_internal_memories(pdev, rproc); + if (ret) + goto free_rproc; + + ret = omap_rproc_get_boot_data(pdev, rproc); + if (ret) + goto free_rproc; + + ret = omap_rproc_of_get_timers(pdev, rproc); + if (ret) + goto free_rproc; + + init_completion(&oproc->pm_comp); + oproc->autosuspend_delay = DEFAULT_AUTOSUSPEND_DELAY; + + of_property_read_u32(pdev->dev.of_node, "ti,autosuspend-delay-ms", + &oproc->autosuspend_delay); + + pm_runtime_set_autosuspend_delay(&pdev->dev, oproc->autosuspend_delay); + + oproc->fck = devm_clk_get(&pdev->dev, 0); + if (IS_ERR(oproc->fck)) { + ret = PTR_ERR(oproc->fck); + goto free_rproc; + } + + ret = of_reserved_mem_device_init(&pdev->dev); + if (ret) { + dev_warn(&pdev->dev, "device does not have specific CMA pool.\n"); + dev_warn(&pdev->dev, "Typically this should be provided,\n"); + dev_warn(&pdev->dev, "only omit if you know what you are doing.\n"); + } + platform_set_drvdata(pdev, rproc); ret = rproc_add(rproc); if (ret) - goto free_rproc; + goto release_mem; return 0; +release_mem: + of_reserved_mem_device_release(&pdev->dev); free_rproc: rproc_free(rproc); return ret; @@ -217,15 +1372,24 @@ static int omap_rproc_remove(struct platform_device *pdev) rproc_del(rproc); rproc_free(rproc); + of_reserved_mem_device_release(&pdev->dev); return 0; } +static const struct dev_pm_ops omap_rproc_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(omap_rproc_suspend, omap_rproc_resume) + SET_RUNTIME_PM_OPS(omap_rproc_runtime_suspend, + omap_rproc_runtime_resume, NULL) +}; + static struct platform_driver omap_rproc_driver = { .probe = omap_rproc_probe, .remove = omap_rproc_remove, .driver = { .name = "omap-rproc", + .pm = &omap_rproc_pm_ops, + .of_match_table = omap_rproc_of_match, }, }; diff --git a/drivers/remoteproc/omap_remoteproc.h b/drivers/remoteproc/omap_remoteproc.h index f6d2036d383d..828e13256c02 100644 --- a/drivers/remoteproc/omap_remoteproc.h +++ b/drivers/remoteproc/omap_remoteproc.h @@ -1,35 +1,10 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ /* * Remote processor messaging * - * Copyright (C) 2011 Texas Instruments, Inc. + * Copyright (C) 2011-2020 Texas Instruments, Inc. * Copyright (C) 2011 Google, Inc. * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name Texas Instruments nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _OMAP_RPMSG_H @@ -56,6 +31,22 @@ * * @RP_MBOX_ABORT_REQUEST: a "please crash" request, used for testing the * recovery mechanism (to some extent). + * + * @RP_MBOX_SUSPEND_AUTO: auto suspend request for the remote processor + * + * @RP_MBOX_SUSPEND_SYSTEM: system suspend request for the remote processor + * + * @RP_MBOX_SUSPEND_ACK: successful response from remote processor for a + * suspend request + * + * @RP_MBOX_SUSPEND_CANCEL: a cancel suspend response from a remote processor + * on a suspend request + * + * Introduce new message definitions if any here. + * + * @RP_MBOX_END_MSG: Indicates end of known/defined messages from remote core + * This should be the last definition. + * */ enum omap_rp_mbox_messages { RP_MBOX_READY = 0xFFFFFF00, @@ -64,6 +55,11 @@ enum omap_rp_mbox_messages { RP_MBOX_ECHO_REQUEST = 0xFFFFFF03, RP_MBOX_ECHO_REPLY = 0xFFFFFF04, RP_MBOX_ABORT_REQUEST = 0xFFFFFF05, + RP_MBOX_SUSPEND_AUTO = 0xFFFFFF10, + RP_MBOX_SUSPEND_SYSTEM = 0xFFFFFF11, + RP_MBOX_SUSPEND_ACK = 0xFFFFFF12, + RP_MBOX_SUSPEND_CANCEL = 0xFFFFFF13, + RP_MBOX_END_MSG = 0xFFFFFF14, }; #endif /* _OMAP_RPMSG_H */ diff --git a/drivers/remoteproc/qcom_q6v5.c b/drivers/remoteproc/qcom_q6v5.c index cb0f4a0be032..111a442c993c 100644 --- a/drivers/remoteproc/qcom_q6v5.c +++ b/drivers/remoteproc/qcom_q6v5.c @@ -15,6 +15,8 @@ #include #include "qcom_q6v5.h" +#define Q6V5_PANIC_DELAY_MS 200 + /** * qcom_q6v5_prepare() - reinitialize the qcom_q6v5 context before start * @q6v5: reference to qcom_q6v5 context to be reinitialized @@ -162,6 +164,24 @@ int qcom_q6v5_request_stop(struct qcom_q6v5 *q6v5) } EXPORT_SYMBOL_GPL(qcom_q6v5_request_stop); +/** + * qcom_q6v5_panic() - panic handler to invoke a stop on the remote + * @q6v5: reference to qcom_q6v5 context + * + * Set the stop bit and sleep in order to allow the remote processor to flush + * its caches etc for post mortem debugging. + * + * Return: 200ms + */ +unsigned long qcom_q6v5_panic(struct qcom_q6v5 *q6v5) +{ + qcom_smem_state_update_bits(q6v5->state, + BIT(q6v5->stop_bit), BIT(q6v5->stop_bit)); + + return Q6V5_PANIC_DELAY_MS; +} +EXPORT_SYMBOL_GPL(qcom_q6v5_panic); + /** * qcom_q6v5_init() - initializer of the q6v5 common struct * @q6v5: handle to be initialized diff --git a/drivers/remoteproc/qcom_q6v5.h b/drivers/remoteproc/qcom_q6v5.h index 7ac92c1e0f49..c4ed887c1499 100644 --- a/drivers/remoteproc/qcom_q6v5.h +++ b/drivers/remoteproc/qcom_q6v5.h @@ -42,5 +42,6 @@ int qcom_q6v5_prepare(struct qcom_q6v5 *q6v5); int qcom_q6v5_unprepare(struct qcom_q6v5 *q6v5); int qcom_q6v5_request_stop(struct qcom_q6v5 *q6v5); int qcom_q6v5_wait_for_start(struct qcom_q6v5 *q6v5, int timeout); +unsigned long qcom_q6v5_panic(struct qcom_q6v5 *q6v5); #endif diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index e953886b2eb7..24a3db961d5e 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -270,7 +270,7 @@ static int adsp_stop(struct rproc *rproc) return ret; } -static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len) +static void *adsp_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv; int offset; @@ -282,12 +282,20 @@ static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len) return adsp->mem_region + offset; } +static unsigned long adsp_panic(struct rproc *rproc) +{ + struct qcom_adsp *adsp = rproc->priv; + + return qcom_q6v5_panic(&adsp->q6v5); +} + static const struct rproc_ops adsp_ops = { .start = adsp_start, .stop = adsp_stop, .da_to_va = adsp_da_to_va, .parse_fw = qcom_register_dump_segments, .load = adsp_load, + .panic = adsp_panic, }; static int adsp_init_clock(struct qcom_adsp *adsp, const char **clk_ids) diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index f9ccce76e44b..ce49c3236ff7 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -381,23 +381,33 @@ static void q6v5_pds_disable(struct q6v5 *qproc, struct device **pds, } static int q6v5_xfer_mem_ownership(struct q6v5 *qproc, int *current_perm, - bool remote_owner, phys_addr_t addr, + bool local, bool remote, phys_addr_t addr, size_t size) { - struct qcom_scm_vmperm next; + struct qcom_scm_vmperm next[2]; + int perms = 0; if (!qproc->need_mem_protection) return 0; - if (remote_owner && *current_perm == BIT(QCOM_SCM_VMID_MSS_MSA)) - return 0; - if (!remote_owner && *current_perm == BIT(QCOM_SCM_VMID_HLOS)) + + if (local == !!(*current_perm & BIT(QCOM_SCM_VMID_HLOS)) && + remote == !!(*current_perm & BIT(QCOM_SCM_VMID_MSS_MSA))) return 0; - next.vmid = remote_owner ? QCOM_SCM_VMID_MSS_MSA : QCOM_SCM_VMID_HLOS; - next.perm = remote_owner ? QCOM_SCM_PERM_RW : QCOM_SCM_PERM_RWX; + if (local) { + next[perms].vmid = QCOM_SCM_VMID_HLOS; + next[perms].perm = QCOM_SCM_PERM_RWX; + perms++; + } + + if (remote) { + next[perms].vmid = QCOM_SCM_VMID_MSS_MSA; + next[perms].perm = QCOM_SCM_PERM_RW; + perms++; + } return qcom_scm_assign_mem(addr, ALIGN(size, SZ_4K), - current_perm, &next, 1); + current_perm, next, perms); } static int q6v5_load(struct rproc *rproc, const struct firmware *fw) @@ -803,7 +813,8 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw) /* Hypervisor mapping to access metadata by modem */ mdata_perm = BIT(QCOM_SCM_VMID_HLOS); - ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, true, phys, size); + ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, false, true, + phys, size); if (ret) { dev_err(qproc->dev, "assigning Q6 access to metadata failed: %d\n", ret); @@ -821,7 +832,8 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw) dev_err(qproc->dev, "MPSS header authentication failed: %d\n", ret); /* Metadata authentication done, remove modem access */ - xferop_ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, false, phys, size); + xferop_ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, true, false, + phys, size); if (xferop_ret) dev_warn(qproc->dev, "mdt buffer not reclaimed system may become unstable\n"); @@ -908,7 +920,7 @@ static int q6v5_mba_load(struct q6v5 *qproc) } /* Assign MBA image access in DDR to q6 */ - ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true, + ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, false, true, qproc->mba_phys, qproc->mba_size); if (ret) { dev_err(qproc->dev, @@ -945,8 +957,8 @@ static int q6v5_mba_load(struct q6v5 *qproc) q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_nc); reclaim_mba: - xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, false, - qproc->mba_phys, + xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true, + false, qproc->mba_phys, qproc->mba_size); if (xfermemop_ret) { dev_err(qproc->dev, @@ -1003,11 +1015,6 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc) writel(val, qproc->reg_base + QDSP6SS_PWR_CTL_REG); } - ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, - false, qproc->mpss_phys, - qproc->mpss_size); - WARN_ON(ret); - q6v5_reset_assert(qproc); q6v5_clk_disable(qproc->dev, qproc->reset_clks, @@ -1021,7 +1028,7 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc) /* In case of failure or coredump scenario where reclaiming MBA memory * could not happen reclaim it here. */ - ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, false, + ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true, false, qproc->mba_phys, qproc->mba_size); WARN_ON(ret); @@ -1037,6 +1044,23 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc) } } +static int q6v5_reload_mba(struct rproc *rproc) +{ + struct q6v5 *qproc = rproc->priv; + const struct firmware *fw; + int ret; + + ret = request_firmware(&fw, rproc->firmware, qproc->dev); + if (ret < 0) + return ret; + + q6v5_load(rproc, fw); + ret = q6v5_mba_load(qproc); + release_firmware(fw); + + return ret; +} + static int q6v5_mpss_load(struct q6v5 *qproc) { const struct elf32_phdr *phdrs; @@ -1048,6 +1072,7 @@ static int q6v5_mpss_load(struct q6v5 *qproc) phys_addr_t boot_addr; phys_addr_t min_addr = PHYS_ADDR_MAX; phys_addr_t max_addr = 0; + u32 code_length; bool relocate = false; char *fw_name; size_t fw_name_len; @@ -1097,6 +1122,24 @@ static int q6v5_mpss_load(struct q6v5 *qproc) max_addr = ALIGN(phdr->p_paddr + phdr->p_memsz, SZ_4K); } + /** + * In case of a modem subsystem restart on secure devices, the modem + * memory can be reclaimed only after MBA is loaded. For modem cold + * boot this will be a nop + */ + q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, true, false, + qproc->mpss_phys, qproc->mpss_size); + + /* Share ownership between Linux and MSS, during segment loading */ + ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, true, true, + qproc->mpss_phys, qproc->mpss_size); + if (ret) { + dev_err(qproc->dev, + "assigning Q6 access to mpss memory failed: %d\n", ret); + ret = -EAGAIN; + goto release_firmware; + } + mpss_reloc = relocate ? min_addr : qproc->mpss_phys; qproc->mpss_reloc = mpss_reloc; /* Load firmware segments */ @@ -1145,10 +1188,25 @@ static int q6v5_mpss_load(struct q6v5 *qproc) phdr->p_memsz - phdr->p_filesz); } size += phdr->p_memsz; + + code_length = readl(qproc->rmb_base + RMB_PMI_CODE_LENGTH_REG); + if (!code_length) { + boot_addr = relocate ? qproc->mpss_phys : min_addr; + writel(boot_addr, qproc->rmb_base + RMB_PMI_CODE_START_REG); + writel(RMB_CMD_LOAD_READY, qproc->rmb_base + RMB_MBA_COMMAND_REG); + } + writel(size, qproc->rmb_base + RMB_PMI_CODE_LENGTH_REG); + + ret = readl(qproc->rmb_base + RMB_MBA_STATUS_REG); + if (ret < 0) { + dev_err(qproc->dev, "MPSS authentication failed: %d\n", + ret); + goto release_firmware; + } } /* Transfer ownership of modem ddr region to q6 */ - ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, true, + ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, false, true, qproc->mpss_phys, qproc->mpss_size); if (ret) { dev_err(qproc->dev, @@ -1157,11 +1215,6 @@ static int q6v5_mpss_load(struct q6v5 *qproc) goto release_firmware; } - boot_addr = relocate ? qproc->mpss_phys : min_addr; - writel(boot_addr, qproc->rmb_base + RMB_PMI_CODE_START_REG); - writel(RMB_CMD_LOAD_READY, qproc->rmb_base + RMB_MBA_COMMAND_REG); - writel(size, qproc->rmb_base + RMB_PMI_CODE_LENGTH_REG); - ret = q6v5_rmb_mba_wait(qproc, RMB_MBA_AUTH_COMPLETE, 10000); if (ret == -ETIMEDOUT) dev_err(qproc->dev, "MPSS authentication timed out\n"); @@ -1186,8 +1239,16 @@ static void qcom_q6v5_dump_segment(struct rproc *rproc, void *ptr = rproc_da_to_va(rproc, segment->da, segment->size); /* Unlock mba before copying segments */ - if (!qproc->dump_mba_loaded) - ret = q6v5_mba_load(qproc); + if (!qproc->dump_mba_loaded) { + ret = q6v5_reload_mba(rproc); + if (!ret) { + /* Reset ownership back to Linux to copy segments */ + ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, + true, false, + qproc->mpss_phys, + qproc->mpss_size); + } + } if (!ptr || ret) memset(dest, 0xff, segment->size); @@ -1198,8 +1259,14 @@ static void qcom_q6v5_dump_segment(struct rproc *rproc, /* Reclaim mba after copying segments */ if (qproc->dump_segment_mask == qproc->dump_complete_mask) { - if (qproc->dump_mba_loaded) + if (qproc->dump_mba_loaded) { + /* Try to reset ownership back to Q6 */ + q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, + false, true, + qproc->mpss_phys, + qproc->mpss_size); q6v5_mba_reclaim(qproc); + } } } @@ -1225,8 +1292,8 @@ static int q6v5_start(struct rproc *rproc) goto reclaim_mpss; } - xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, false, - qproc->mba_phys, + xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mba_perm, true, + false, qproc->mba_phys, qproc->mba_size); if (xfermemop_ret) dev_err(qproc->dev, @@ -1239,10 +1306,6 @@ static int q6v5_start(struct rproc *rproc) return 0; reclaim_mpss: - xfermemop_ret = q6v5_xfer_mem_ownership(qproc, &qproc->mpss_perm, - false, qproc->mpss_phys, - qproc->mpss_size); - WARN_ON(xfermemop_ret); q6v5_mba_reclaim(qproc); return ret; @@ -1264,7 +1327,7 @@ static int q6v5_stop(struct rproc *rproc) return 0; } -static void *q6v5_da_to_va(struct rproc *rproc, u64 da, int len) +static void *q6v5_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct q6v5 *qproc = rproc->priv; int offset; diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index edf9d0e1a235..7a63efb85405 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -222,7 +222,7 @@ static int adsp_stop(struct rproc *rproc) return ret; } -static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len) +static void *adsp_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv; int offset; @@ -234,12 +234,20 @@ static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len) return adsp->mem_region + offset; } +static unsigned long adsp_panic(struct rproc *rproc) +{ + struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv; + + return qcom_q6v5_panic(&adsp->q6v5); +} + static const struct rproc_ops adsp_ops = { .start = adsp_start, .stop = adsp_stop, .da_to_va = adsp_da_to_va, .parse_fw = qcom_register_dump_segments, .load = adsp_load, + .panic = adsp_panic, }; static int adsp_init_clock(struct qcom_adsp *adsp) diff --git a/drivers/remoteproc/qcom_q6v5_wcss.c b/drivers/remoteproc/qcom_q6v5_wcss.c index f93e1e4a1cc0..f1924b740a10 100644 --- a/drivers/remoteproc/qcom_q6v5_wcss.c +++ b/drivers/remoteproc/qcom_q6v5_wcss.c @@ -406,7 +406,7 @@ static int q6v5_wcss_stop(struct rproc *rproc) return 0; } -static void *q6v5_wcss_da_to_va(struct rproc *rproc, u64 da, int len) +static void *q6v5_wcss_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct q6v5_wcss *wcss = rproc->priv; int offset; diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index dc135754bb9c..0c7afd038f0d 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -287,7 +287,7 @@ static int wcnss_stop(struct rproc *rproc) return ret; } -static void *wcnss_da_to_va(struct rproc *rproc, u64 da, int len) +static void *wcnss_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv; int offset; diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 097f33e4f1f3..e12a54e67588 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -16,6 +16,7 @@ #define pr_fmt(fmt) "%s: " fmt, __func__ +#include #include #include #include @@ -26,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -38,11 +40,13 @@ #include #include "remoteproc_internal.h" +#include "remoteproc_elf_helpers.h" #define HIGH_BITS_MASK 0xFFFFFFFF00000000ULL static DEFINE_MUTEX(rproc_list_mutex); static LIST_HEAD(rproc_list); +static struct notifier_block rproc_panic_nb; typedef int (*rproc_handle_resource_t)(struct rproc *rproc, void *, int offset, int avail); @@ -185,7 +189,7 @@ EXPORT_SYMBOL(rproc_va_to_pa); * here the output of the DMA API for the carveouts, which should be more * correct. */ -void *rproc_da_to_va(struct rproc *rproc, u64 da, int len) +void *rproc_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct rproc_mem_entry *carveout; void *ptr = NULL; @@ -224,7 +228,8 @@ EXPORT_SYMBOL(rproc_da_to_va); /** * rproc_find_carveout_by_name() - lookup the carveout region by a name * @rproc: handle of a remote processor - * @name,..: carveout name to find (standard printf format) + * @name: carveout name to find (format string) + * @...: optional parameters matching @name string * * Platform driver has the capability to register some pre-allacoted carveout * (physically contiguous memory regions) before rproc firmware loading and @@ -318,8 +323,9 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i) struct device *dev = &rproc->dev; struct rproc_vring *rvring = &rvdev->vring[i]; struct fw_rsc_vdev *rsc; - int ret, size, notifyid; + int ret, notifyid; struct rproc_mem_entry *mem; + size_t size; /* actual size of vring (in bytes) */ size = PAGE_ALIGN(vring_size(rvring->len, rvring->align)); @@ -445,6 +451,7 @@ static void rproc_rvdev_release(struct device *dev) * rproc_handle_vdev() - handle a vdev fw resource * @rproc: the remote processor * @rsc: the vring resource descriptor + * @offset: offset of the resource entry * @avail: size of available data (for sanity checking the image) * * This resource entry requests the host to statically register a virtio @@ -587,6 +594,7 @@ void rproc_vdev_release(struct kref *ref) * rproc_handle_trace() - handle a shared trace buffer resource * @rproc: the remote processor * @rsc: the trace resource descriptor + * @offset: offset of the resource entry * @avail: size of available data (for sanity checking the image) * * In case the remote processor dumps trace logs into memory, @@ -652,6 +660,7 @@ static int rproc_handle_trace(struct rproc *rproc, struct fw_rsc_trace *rsc, * rproc_handle_devmem() - handle devmem resource entry * @rproc: remote processor handle * @rsc: the devmem resource entry + * @offset: offset of the resource entry * @avail: size of available data (for sanity checking the image) * * Remote processors commonly need to access certain on-chip peripherals. @@ -746,11 +755,12 @@ static int rproc_alloc_carveout(struct rproc *rproc, va = dma_alloc_coherent(dev->parent, mem->len, &dma, GFP_KERNEL); if (!va) { dev_err(dev->parent, - "failed to allocate dma memory: len 0x%x\n", mem->len); + "failed to allocate dma memory: len 0x%zx\n", + mem->len); return -ENOMEM; } - dev_dbg(dev, "carveout va %pK, dma %pad, len 0x%x\n", + dev_dbg(dev, "carveout va %pK, dma %pad, len 0x%zx\n", va, &dma, mem->len); if (mem->da != FW_RSC_ADDR_ANY && !rproc->domain) { @@ -853,6 +863,7 @@ static int rproc_release_carveout(struct rproc *rproc, * rproc_handle_carveout() - handle phys contig memory allocation requests * @rproc: rproc handle * @rsc: the resource entry + * @offset: offset of the resource entry * @avail: size of available data (for image validation) * * This function will handle firmware requests for allocation of physically @@ -957,7 +968,7 @@ EXPORT_SYMBOL(rproc_add_carveout); */ struct rproc_mem_entry * rproc_mem_entry_init(struct device *dev, - void *va, dma_addr_t dma, int len, u32 da, + void *va, dma_addr_t dma, size_t len, u32 da, int (*alloc)(struct rproc *, struct rproc_mem_entry *), int (*release)(struct rproc *, struct rproc_mem_entry *), const char *name, ...) @@ -999,7 +1010,7 @@ EXPORT_SYMBOL(rproc_mem_entry_init); * provided by client. */ struct rproc_mem_entry * -rproc_of_resm_mem_entry_init(struct device *dev, u32 of_resm_idx, int len, +rproc_of_resm_mem_entry_init(struct device *dev, u32 of_resm_idx, size_t len, u32 da, const char *name, ...) { struct rproc_mem_entry *mem; @@ -1022,7 +1033,7 @@ rproc_of_resm_mem_entry_init(struct device *dev, u32 of_resm_idx, int len, } EXPORT_SYMBOL(rproc_of_resm_mem_entry_init); -/** +/* * A lookup table for resource handlers. The indices are defined in * enum fw_resource_type. */ @@ -1270,7 +1281,7 @@ static void rproc_resource_cleanup(struct rproc *rproc) unmapped = iommu_unmap(rproc->domain, entry->da, entry->len); if (unmapped != entry->len) { /* nothing much to do besides complaining */ - dev_err(dev, "failed to unmap %u/%zu\n", entry->len, + dev_err(dev, "failed to unmap %zx/%zu\n", entry->len, unmapped); } @@ -1564,20 +1575,21 @@ EXPORT_SYMBOL(rproc_coredump_add_custom_segment); static void rproc_coredump(struct rproc *rproc) { struct rproc_dump_segment *segment; - struct elf32_phdr *phdr; - struct elf32_hdr *ehdr; + void *phdr; + void *ehdr; size_t data_size; size_t offset; void *data; void *ptr; + u8 class = rproc->elf_class; int phnum = 0; if (list_empty(&rproc->dump_segments)) return; - data_size = sizeof(*ehdr); + data_size = elf_size_of_hdr(class); list_for_each_entry(segment, &rproc->dump_segments, node) { - data_size += sizeof(*phdr) + segment->size; + data_size += elf_size_of_phdr(class) + segment->size; phnum++; } @@ -1588,33 +1600,33 @@ static void rproc_coredump(struct rproc *rproc) ehdr = data; - memset(ehdr, 0, sizeof(*ehdr)); - memcpy(ehdr->e_ident, ELFMAG, SELFMAG); - ehdr->e_ident[EI_CLASS] = ELFCLASS32; - ehdr->e_ident[EI_DATA] = ELFDATA2LSB; - ehdr->e_ident[EI_VERSION] = EV_CURRENT; - ehdr->e_ident[EI_OSABI] = ELFOSABI_NONE; - ehdr->e_type = ET_CORE; - ehdr->e_machine = EM_NONE; - ehdr->e_version = EV_CURRENT; - ehdr->e_entry = rproc->bootaddr; - ehdr->e_phoff = sizeof(*ehdr); - ehdr->e_ehsize = sizeof(*ehdr); - ehdr->e_phentsize = sizeof(*phdr); - ehdr->e_phnum = phnum; + memset(ehdr, 0, elf_size_of_hdr(class)); + /* e_ident field is common for both elf32 and elf64 */ + elf_hdr_init_ident(ehdr, class); + + elf_hdr_set_e_type(class, ehdr, ET_CORE); + elf_hdr_set_e_machine(class, ehdr, EM_NONE); + elf_hdr_set_e_version(class, ehdr, EV_CURRENT); + elf_hdr_set_e_entry(class, ehdr, rproc->bootaddr); + elf_hdr_set_e_phoff(class, ehdr, elf_size_of_hdr(class)); + elf_hdr_set_e_ehsize(class, ehdr, elf_size_of_hdr(class)); + elf_hdr_set_e_phentsize(class, ehdr, elf_size_of_phdr(class)); + elf_hdr_set_e_phnum(class, ehdr, phnum); + + phdr = data + elf_hdr_get_e_phoff(class, ehdr); + offset = elf_hdr_get_e_phoff(class, ehdr); + offset += elf_size_of_phdr(class) * elf_hdr_get_e_phnum(class, ehdr); - phdr = data + ehdr->e_phoff; - offset = ehdr->e_phoff + sizeof(*phdr) * ehdr->e_phnum; list_for_each_entry(segment, &rproc->dump_segments, node) { - memset(phdr, 0, sizeof(*phdr)); - phdr->p_type = PT_LOAD; - phdr->p_offset = offset; - phdr->p_vaddr = segment->da; - phdr->p_paddr = segment->da; - phdr->p_filesz = segment->size; - phdr->p_memsz = segment->size; - phdr->p_flags = PF_R | PF_W | PF_X; - phdr->p_align = 0; + memset(phdr, 0, elf_size_of_phdr(class)); + elf_phdr_set_p_type(class, phdr, PT_LOAD); + elf_phdr_set_p_offset(class, phdr, offset); + elf_phdr_set_p_vaddr(class, phdr, segment->da); + elf_phdr_set_p_paddr(class, phdr, segment->da); + elf_phdr_set_p_filesz(class, phdr, segment->size); + elf_phdr_set_p_memsz(class, phdr, segment->size); + elf_phdr_set_p_flags(class, phdr, PF_R | PF_W | PF_X); + elf_phdr_set_p_align(class, phdr, 0); if (segment->dump) { segment->dump(rproc, segment, data + offset); @@ -1630,8 +1642,8 @@ static void rproc_coredump(struct rproc *rproc) } } - offset += phdr->p_filesz; - phdr++; + offset += elf_phdr_get_p_filesz(class, phdr); + phdr += elf_size_of_phdr(class); } dev_coredumpv(&rproc->dev, data, data_size, GFP_KERNEL); @@ -1653,12 +1665,16 @@ int rproc_trigger_recovery(struct rproc *rproc) struct device *dev = &rproc->dev; int ret; - dev_err(dev, "recovering %s\n", rproc->name); - ret = mutex_lock_interruptible(&rproc->lock); if (ret) return ret; + /* State could have changed before we got the mutex */ + if (rproc->state != RPROC_CRASHED) + goto unlock_mutex; + + dev_err(dev, "recovering %s\n", rproc->name); + ret = rproc_stop(rproc, true); if (ret) goto unlock_mutex; @@ -1685,6 +1701,7 @@ int rproc_trigger_recovery(struct rproc *rproc) /** * rproc_crash_handler_work() - handle a crash + * @work: work treating the crash * * This function needs to handle everything related to a crash, like cpu * registers and stack dump, information to help to debug the fatal error, etc. @@ -1854,8 +1871,8 @@ struct rproc *rproc_get_by_phandle(phandle phandle) if (!np) return NULL; - mutex_lock(&rproc_list_mutex); - list_for_each_entry(r, &rproc_list, node) { + rcu_read_lock(); + list_for_each_entry_rcu(r, &rproc_list, node) { if (r->dev.parent && r->dev.parent->of_node == np) { /* prevent underlying implementation from being removed */ if (!try_module_get(r->dev.parent->driver->owner)) { @@ -1868,7 +1885,7 @@ struct rproc *rproc_get_by_phandle(phandle phandle) break; } } - mutex_unlock(&rproc_list_mutex); + rcu_read_unlock(); of_node_put(np); @@ -1925,7 +1942,7 @@ int rproc_add(struct rproc *rproc) /* expose to rproc_get_by_phandle users */ mutex_lock(&rproc_list_mutex); - list_add(&rproc->node, &rproc_list); + list_add_rcu(&rproc->node, &rproc_list); mutex_unlock(&rproc_list_mutex); return 0; @@ -2029,6 +2046,7 @@ struct rproc *rproc_alloc(struct device *dev, const char *name, rproc->name = name; rproc->priv = &rproc[1]; rproc->auto_boot = true; + rproc->elf_class = ELFCLASS32; device_initialize(&rproc->dev); rproc->dev.parent = dev; @@ -2053,7 +2071,8 @@ struct rproc *rproc_alloc(struct device *dev, const char *name, rproc->ops->load = rproc_elf_load_segments; rproc->ops->parse_fw = rproc_elf_load_rsc_table; rproc->ops->find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table; - rproc->ops->sanity_check = rproc_elf_sanity_check; + if (!rproc->ops->sanity_check) + rproc->ops->sanity_check = rproc_elf32_sanity_check; rproc->ops->get_boot_addr = rproc_elf_get_boot_addr; } @@ -2140,9 +2159,12 @@ int rproc_del(struct rproc *rproc) /* the rproc is downref'ed as soon as it's removed from the klist */ mutex_lock(&rproc_list_mutex); - list_del(&rproc->node); + list_del_rcu(&rproc->node); mutex_unlock(&rproc_list_mutex); + /* Ensure that no readers of rproc_list are still active */ + synchronize_rcu(); + device_del(&rproc->dev); return 0; @@ -2216,10 +2238,50 @@ void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type) } EXPORT_SYMBOL(rproc_report_crash); +static int rproc_panic_handler(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + unsigned int longest = 0; + struct rproc *rproc; + unsigned int d; + + rcu_read_lock(); + list_for_each_entry_rcu(rproc, &rproc_list, node) { + if (!rproc->ops->panic || rproc->state != RPROC_RUNNING) + continue; + + d = rproc->ops->panic(rproc); + longest = max(longest, d); + } + rcu_read_unlock(); + + /* + * Delay for the longest requested duration before returning. This can + * be used by the remoteproc drivers to give the remote processor time + * to perform any requested operations (such as flush caches), when + * it's not possible to signal the Linux side due to the panic. + */ + mdelay(longest); + + return NOTIFY_DONE; +} + +static void __init rproc_init_panic(void) +{ + rproc_panic_nb.notifier_call = rproc_panic_handler; + atomic_notifier_chain_register(&panic_notifier_list, &rproc_panic_nb); +} + +static void __exit rproc_exit_panic(void) +{ + atomic_notifier_chain_unregister(&panic_notifier_list, &rproc_panic_nb); +} + static int __init remoteproc_init(void) { rproc_init_sysfs(); rproc_init_debugfs(); + rproc_init_panic(); return 0; } @@ -2229,6 +2291,7 @@ static void __exit remoteproc_exit(void) { ida_destroy(&rproc_dev_index); + rproc_exit_panic(); rproc_exit_debugfs(); rproc_exit_sysfs(); } diff --git a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c index dd93cf04e17f..d734cadb16e3 100644 --- a/drivers/remoteproc/remoteproc_debugfs.c +++ b/drivers/remoteproc/remoteproc_debugfs.c @@ -138,16 +138,16 @@ rproc_recovery_write(struct file *filp, const char __user *user_buf, buf[count - 1] = '\0'; if (!strncmp(buf, "enabled", count)) { + /* change the flag and begin the recovery process if needed */ rproc->recovery_disabled = false; - /* if rproc has crashed, trigger recovery */ - if (rproc->state == RPROC_CRASHED) - rproc_trigger_recovery(rproc); + rproc_trigger_recovery(rproc); } else if (!strncmp(buf, "disabled", count)) { rproc->recovery_disabled = true; } else if (!strncmp(buf, "recover", count)) { - /* if rproc has crashed, trigger recovery */ - if (rproc->state == RPROC_CRASHED) - rproc_trigger_recovery(rproc); + /* begin the recovery process without changing the flag */ + rproc_trigger_recovery(rproc); + } else { + return -EINVAL; } return count; @@ -293,7 +293,7 @@ static int rproc_carveouts_show(struct seq_file *seq, void *p) seq_printf(seq, "\tVirtual address: %pK\n", carveout->va); seq_printf(seq, "\tDMA address: %pad\n", &carveout->dma); seq_printf(seq, "\tDevice address: 0x%x\n", carveout->da); - seq_printf(seq, "\tLength: 0x%x Bytes\n\n", carveout->len); + seq_printf(seq, "\tLength: 0x%zx Bytes\n\n", carveout->len); } return 0; @@ -349,7 +349,7 @@ void rproc_create_debug_dir(struct rproc *rproc) debugfs_create_file("name", 0400, rproc->dbg_dir, rproc, &rproc_name_ops); - debugfs_create_file("recovery", 0400, rproc->dbg_dir, + debugfs_create_file("recovery", 0600, rproc->dbg_dir, rproc, &rproc_recovery_ops); debugfs_create_file("crash", 0200, rproc->dbg_dir, rproc, &rproc_crash_ops); diff --git a/drivers/remoteproc/remoteproc_elf_helpers.h b/drivers/remoteproc/remoteproc_elf_helpers.h new file mode 100644 index 000000000000..4b6be7b6bf4d --- /dev/null +++ b/drivers/remoteproc/remoteproc_elf_helpers.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Remote processor elf helpers defines + * + * Copyright (C) 2020 Kalray, Inc. + */ + +#ifndef REMOTEPROC_ELF_LOADER_H +#define REMOTEPROC_ELF_LOADER_H + +#include +#include + +/** + * fw_elf_get_class - Get elf class + * @fw: the ELF firmware image + * + * Note that we use and elf32_hdr to access the class since the start of the + * struct is the same for both elf class + * + * Return: elf class of the firmware + */ +static inline u8 fw_elf_get_class(const struct firmware *fw) +{ + struct elf32_hdr *ehdr = (struct elf32_hdr *)fw->data; + + return ehdr->e_ident[EI_CLASS]; +} + +static inline void elf_hdr_init_ident(struct elf32_hdr *hdr, u8 class) +{ + memcpy(hdr->e_ident, ELFMAG, SELFMAG); + hdr->e_ident[EI_CLASS] = class; + hdr->e_ident[EI_DATA] = ELFDATA2LSB; + hdr->e_ident[EI_VERSION] = EV_CURRENT; + hdr->e_ident[EI_OSABI] = ELFOSABI_NONE; +} + +/* Generate getter and setter for a specific elf struct/field */ +#define ELF_GEN_FIELD_GET_SET(__s, __field, __type) \ +static inline __type elf_##__s##_get_##__field(u8 class, const void *arg) \ +{ \ + if (class == ELFCLASS32) \ + return (__type) ((const struct elf32_##__s *) arg)->__field; \ + else \ + return (__type) ((const struct elf64_##__s *) arg)->__field; \ +} \ +static inline void elf_##__s##_set_##__field(u8 class, void *arg, \ + __type value) \ +{ \ + if (class == ELFCLASS32) \ + ((struct elf32_##__s *) arg)->__field = (__type) value; \ + else \ + ((struct elf64_##__s *) arg)->__field = (__type) value; \ +} + +ELF_GEN_FIELD_GET_SET(hdr, e_entry, u64) +ELF_GEN_FIELD_GET_SET(hdr, e_phnum, u16) +ELF_GEN_FIELD_GET_SET(hdr, e_shnum, u16) +ELF_GEN_FIELD_GET_SET(hdr, e_phoff, u64) +ELF_GEN_FIELD_GET_SET(hdr, e_shoff, u64) +ELF_GEN_FIELD_GET_SET(hdr, e_shstrndx, u16) +ELF_GEN_FIELD_GET_SET(hdr, e_machine, u16) +ELF_GEN_FIELD_GET_SET(hdr, e_type, u16) +ELF_GEN_FIELD_GET_SET(hdr, e_version, u32) +ELF_GEN_FIELD_GET_SET(hdr, e_ehsize, u32) +ELF_GEN_FIELD_GET_SET(hdr, e_phentsize, u16) + +ELF_GEN_FIELD_GET_SET(phdr, p_paddr, u64) +ELF_GEN_FIELD_GET_SET(phdr, p_vaddr, u64) +ELF_GEN_FIELD_GET_SET(phdr, p_filesz, u64) +ELF_GEN_FIELD_GET_SET(phdr, p_memsz, u64) +ELF_GEN_FIELD_GET_SET(phdr, p_type, u32) +ELF_GEN_FIELD_GET_SET(phdr, p_offset, u64) +ELF_GEN_FIELD_GET_SET(phdr, p_flags, u32) +ELF_GEN_FIELD_GET_SET(phdr, p_align, u64) + +ELF_GEN_FIELD_GET_SET(shdr, sh_size, u64) +ELF_GEN_FIELD_GET_SET(shdr, sh_offset, u64) +ELF_GEN_FIELD_GET_SET(shdr, sh_name, u32) +ELF_GEN_FIELD_GET_SET(shdr, sh_addr, u64) + +#define ELF_STRUCT_SIZE(__s) \ +static inline unsigned long elf_size_of_##__s(u8 class) \ +{ \ + if (class == ELFCLASS32)\ + return sizeof(struct elf32_##__s); \ + else \ + return sizeof(struct elf64_##__s); \ +} + +ELF_STRUCT_SIZE(shdr) +ELF_STRUCT_SIZE(phdr) +ELF_STRUCT_SIZE(hdr) + +#endif /* REMOTEPROC_ELF_LOADER_H */ diff --git a/drivers/remoteproc/remoteproc_elf_loader.c b/drivers/remoteproc/remoteproc_elf_loader.c index 606aae166eba..16e2c496fd45 100644 --- a/drivers/remoteproc/remoteproc_elf_loader.c +++ b/drivers/remoteproc/remoteproc_elf_loader.c @@ -23,20 +23,29 @@ #include #include "remoteproc_internal.h" +#include "remoteproc_elf_helpers.h" /** - * rproc_elf_sanity_check() - Sanity Check ELF firmware image + * rproc_elf_sanity_check() - Sanity Check for ELF32/ELF64 firmware image * @rproc: the remote processor handle * @fw: the ELF firmware image * - * Make sure this fw image is sane. + * Make sure this fw image is sane (ie a correct ELF32/ELF64 file). */ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw) { const char *name = rproc->firmware; struct device *dev = &rproc->dev; + /* + * Elf files are beginning with the same structure. Thus, to simplify + * header parsing, we can use the elf32_hdr one for both elf64 and + * elf32. + */ struct elf32_hdr *ehdr; + u32 elf_shdr_get_size; + u64 phoff, shoff; char class; + u16 phnum; if (!fw) { dev_err(dev, "failed to load %s\n", name); @@ -50,13 +59,22 @@ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw) ehdr = (struct elf32_hdr *)fw->data; - /* We only support ELF32 at this point */ + if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG)) { + dev_err(dev, "Image is corrupted (bad magic)\n"); + return -EINVAL; + } + class = ehdr->e_ident[EI_CLASS]; - if (class != ELFCLASS32) { + if (class != ELFCLASS32 && class != ELFCLASS64) { dev_err(dev, "Unsupported class: %d\n", class); return -EINVAL; } + if (class == ELFCLASS64 && fw->size < sizeof(struct elf64_hdr)) { + dev_err(dev, "elf64 header is too small\n"); + return -EINVAL; + } + /* We assume the firmware has the same endianness as the host */ # ifdef __LITTLE_ENDIAN if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB) { @@ -67,30 +85,54 @@ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw) return -EINVAL; } - if (fw->size < ehdr->e_shoff + sizeof(struct elf32_shdr)) { + phoff = elf_hdr_get_e_phoff(class, fw->data); + shoff = elf_hdr_get_e_shoff(class, fw->data); + phnum = elf_hdr_get_e_phnum(class, fw->data); + elf_shdr_get_size = elf_size_of_shdr(class); + + if (fw->size < shoff + elf_shdr_get_size) { dev_err(dev, "Image is too small\n"); return -EINVAL; } - if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG)) { - dev_err(dev, "Image is corrupted (bad magic)\n"); - return -EINVAL; - } - - if (ehdr->e_phnum == 0) { + if (phnum == 0) { dev_err(dev, "No loadable segments\n"); return -EINVAL; } - if (ehdr->e_phoff > fw->size) { + if (phoff > fw->size) { dev_err(dev, "Firmware size is too small\n"); return -EINVAL; } + dev_dbg(dev, "Firmware is an elf%d file\n", + class == ELFCLASS32 ? 32 : 64); + return 0; } EXPORT_SYMBOL(rproc_elf_sanity_check); +/** + * rproc_elf_sanity_check() - Sanity Check ELF32 firmware image + * @rproc: the remote processor handle + * @fw: the ELF32 firmware image + * + * Make sure this fw image is sane. + */ +int rproc_elf32_sanity_check(struct rproc *rproc, const struct firmware *fw) +{ + int ret = rproc_elf_sanity_check(rproc, fw); + + if (ret) + return ret; + + if (fw_elf_get_class(fw) == ELFCLASS32) + return 0; + + return -EINVAL; +} +EXPORT_SYMBOL(rproc_elf32_sanity_check); + /** * rproc_elf_get_boot_addr() - Get rproc's boot address. * @rproc: the remote processor handle @@ -102,11 +144,9 @@ EXPORT_SYMBOL(rproc_elf_sanity_check); * Note that the boot address is not a configurable property of all remote * processors. Some will always boot at a specific hard-coded address. */ -u32 rproc_elf_get_boot_addr(struct rproc *rproc, const struct firmware *fw) +u64 rproc_elf_get_boot_addr(struct rproc *rproc, const struct firmware *fw) { - struct elf32_hdr *ehdr = (struct elf32_hdr *)fw->data; - - return ehdr->e_entry; + return elf_hdr_get_e_entry(fw_elf_get_class(fw), fw->data); } EXPORT_SYMBOL(rproc_elf_get_boot_addr); @@ -137,53 +177,65 @@ EXPORT_SYMBOL(rproc_elf_get_boot_addr); int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw) { struct device *dev = &rproc->dev; - struct elf32_hdr *ehdr; - struct elf32_phdr *phdr; + const void *ehdr, *phdr; int i, ret = 0; + u16 phnum; const u8 *elf_data = fw->data; + u8 class = fw_elf_get_class(fw); + u32 elf_phdr_get_size = elf_size_of_phdr(class); - ehdr = (struct elf32_hdr *)elf_data; - phdr = (struct elf32_phdr *)(elf_data + ehdr->e_phoff); + ehdr = elf_data; + phnum = elf_hdr_get_e_phnum(class, ehdr); + phdr = elf_data + elf_hdr_get_e_phoff(class, ehdr); /* go through the available ELF segments */ - for (i = 0; i < ehdr->e_phnum; i++, phdr++) { - u32 da = phdr->p_paddr; - u32 memsz = phdr->p_memsz; - u32 filesz = phdr->p_filesz; - u32 offset = phdr->p_offset; + for (i = 0; i < phnum; i++, phdr += elf_phdr_get_size) { + u64 da = elf_phdr_get_p_paddr(class, phdr); + u64 memsz = elf_phdr_get_p_memsz(class, phdr); + u64 filesz = elf_phdr_get_p_filesz(class, phdr); + u64 offset = elf_phdr_get_p_offset(class, phdr); + u32 type = elf_phdr_get_p_type(class, phdr); void *ptr; - if (phdr->p_type != PT_LOAD) + if (type != PT_LOAD) continue; - dev_dbg(dev, "phdr: type %d da 0x%x memsz 0x%x filesz 0x%x\n", - phdr->p_type, da, memsz, filesz); + dev_dbg(dev, "phdr: type %d da 0x%llx memsz 0x%llx filesz 0x%llx\n", + type, da, memsz, filesz); if (filesz > memsz) { - dev_err(dev, "bad phdr filesz 0x%x memsz 0x%x\n", + dev_err(dev, "bad phdr filesz 0x%llx memsz 0x%llx\n", filesz, memsz); ret = -EINVAL; break; } if (offset + filesz > fw->size) { - dev_err(dev, "truncated fw: need 0x%x avail 0x%zx\n", + dev_err(dev, "truncated fw: need 0x%llx avail 0x%zx\n", offset + filesz, fw->size); ret = -EINVAL; break; } + if (!rproc_u64_fit_in_size_t(memsz)) { + dev_err(dev, "size (%llx) does not fit in size_t type\n", + memsz); + ret = -EOVERFLOW; + break; + } + /* grab the kernel address for this device address */ ptr = rproc_da_to_va(rproc, da, memsz); if (!ptr) { - dev_err(dev, "bad phdr da 0x%x mem 0x%x\n", da, memsz); + dev_err(dev, "bad phdr da 0x%llx mem 0x%llx\n", da, + memsz); ret = -EINVAL; break; } /* put the segment where the remote processor expects it */ - if (phdr->p_filesz) - memcpy(ptr, elf_data + phdr->p_offset, filesz); + if (filesz) + memcpy(ptr, elf_data + offset, filesz); /* * Zero out remaining memory for this segment. @@ -196,28 +248,42 @@ int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw) memset(ptr + filesz, 0, memsz - filesz); } + if (ret == 0) + rproc->elf_class = class; + return ret; } EXPORT_SYMBOL(rproc_elf_load_segments); -static struct elf32_shdr * -find_table(struct device *dev, struct elf32_hdr *ehdr, size_t fw_size) +static const void * +find_table(struct device *dev, const struct firmware *fw) { - struct elf32_shdr *shdr; + const void *shdr, *name_table_shdr; int i; const char *name_table; struct resource_table *table = NULL; - const u8 *elf_data = (void *)ehdr; + const u8 *elf_data = (void *)fw->data; + u8 class = fw_elf_get_class(fw); + size_t fw_size = fw->size; + const void *ehdr = elf_data; + u16 shnum = elf_hdr_get_e_shnum(class, ehdr); + u32 elf_shdr_get_size = elf_size_of_shdr(class); + u16 shstrndx = elf_hdr_get_e_shstrndx(class, ehdr); /* look for the resource table and handle it */ - shdr = (struct elf32_shdr *)(elf_data + ehdr->e_shoff); - name_table = elf_data + shdr[ehdr->e_shstrndx].sh_offset; + /* First, get the section header according to the elf class */ + shdr = elf_data + elf_hdr_get_e_shoff(class, ehdr); + /* Compute name table section header entry in shdr array */ + name_table_shdr = shdr + (shstrndx * elf_shdr_get_size); + /* Finally, compute the name table section address in elf */ + name_table = elf_data + elf_shdr_get_sh_offset(class, name_table_shdr); - for (i = 0; i < ehdr->e_shnum; i++, shdr++) { - u32 size = shdr->sh_size; - u32 offset = shdr->sh_offset; + for (i = 0; i < shnum; i++, shdr += elf_shdr_get_size) { + u64 size = elf_shdr_get_sh_size(class, shdr); + u64 offset = elf_shdr_get_sh_offset(class, shdr); + u32 name = elf_shdr_get_sh_name(class, shdr); - if (strcmp(name_table + shdr->sh_name, ".resource_table")) + if (strcmp(name_table + name, ".resource_table")) continue; table = (struct resource_table *)(elf_data + offset); @@ -270,21 +336,21 @@ find_table(struct device *dev, struct elf32_hdr *ehdr, size_t fw_size) */ int rproc_elf_load_rsc_table(struct rproc *rproc, const struct firmware *fw) { - struct elf32_hdr *ehdr; - struct elf32_shdr *shdr; + const void *shdr; struct device *dev = &rproc->dev; struct resource_table *table = NULL; const u8 *elf_data = fw->data; size_t tablesz; + u8 class = fw_elf_get_class(fw); + u64 sh_offset; - ehdr = (struct elf32_hdr *)elf_data; - - shdr = find_table(dev, ehdr, fw->size); + shdr = find_table(dev, fw); if (!shdr) return -EINVAL; - table = (struct resource_table *)(elf_data + shdr->sh_offset); - tablesz = shdr->sh_size; + sh_offset = elf_shdr_get_sh_offset(class, shdr); + table = (struct resource_table *)(elf_data + sh_offset); + tablesz = elf_shdr_get_sh_size(class, shdr); /* * Create a copy of the resource table. When a virtio device starts @@ -317,13 +383,24 @@ EXPORT_SYMBOL(rproc_elf_load_rsc_table); struct resource_table *rproc_elf_find_loaded_rsc_table(struct rproc *rproc, const struct firmware *fw) { - struct elf32_hdr *ehdr = (struct elf32_hdr *)fw->data; - struct elf32_shdr *shdr; + const void *shdr; + u64 sh_addr, sh_size; + u8 class = fw_elf_get_class(fw); + struct device *dev = &rproc->dev; - shdr = find_table(&rproc->dev, ehdr, fw->size); + shdr = find_table(&rproc->dev, fw); if (!shdr) return NULL; - return rproc_da_to_va(rproc, shdr->sh_addr, shdr->sh_size); + sh_addr = elf_shdr_get_sh_addr(class, shdr); + sh_size = elf_shdr_get_sh_size(class, shdr); + + if (!rproc_u64_fit_in_size_t(sh_size)) { + dev_err(dev, "size (%llx) does not fit in size_t type\n", + sh_size); + return NULL; + } + + return rproc_da_to_va(rproc, sh_addr, sh_size); } EXPORT_SYMBOL(rproc_elf_find_loaded_rsc_table); diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h index 493ef9262411..b389dc79da81 100644 --- a/drivers/remoteproc/remoteproc_internal.h +++ b/drivers/remoteproc/remoteproc_internal.h @@ -50,12 +50,13 @@ void rproc_exit_sysfs(void); void rproc_free_vring(struct rproc_vring *rvring); int rproc_alloc_vring(struct rproc_vdev *rvdev, int i); -void *rproc_da_to_va(struct rproc *rproc, u64 da, int len); +void *rproc_da_to_va(struct rproc *rproc, u64 da, size_t len); phys_addr_t rproc_va_to_pa(void *cpu_addr); int rproc_trigger_recovery(struct rproc *rproc); +int rproc_elf32_sanity_check(struct rproc *rproc, const struct firmware *fw); int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw); -u32 rproc_elf_get_boot_addr(struct rproc *rproc, const struct firmware *fw); +u64 rproc_elf_get_boot_addr(struct rproc *rproc, const struct firmware *fw); int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw); int rproc_elf_load_rsc_table(struct rproc *rproc, const struct firmware *fw); struct resource_table *rproc_elf_find_loaded_rsc_table(struct rproc *rproc, @@ -73,7 +74,7 @@ int rproc_fw_sanity_check(struct rproc *rproc, const struct firmware *fw) } static inline -u32 rproc_get_boot_addr(struct rproc *rproc, const struct firmware *fw) +u64 rproc_get_boot_addr(struct rproc *rproc, const struct firmware *fw) { if (rproc->ops->get_boot_addr) return rproc->ops->get_boot_addr(rproc, fw); @@ -119,4 +120,13 @@ struct resource_table *rproc_find_loaded_rsc_table(struct rproc *rproc, return NULL; } +static inline +bool rproc_u64_fit_in_size_t(u64 val) +{ + if (sizeof(size_t) == sizeof(u64)) + return true; + + return (val <= (size_t) -1); +} + #endif /* REMOTEPROC_INTERNAL_H */ diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 8c07cb2ca8ba..e61d738d9b47 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -320,6 +320,7 @@ static void rproc_virtio_dev_release(struct device *dev) /** * rproc_add_virtio_dev() - register an rproc-induced virtio device * @rvdev: the remote vdev + * @id: the device type identification (used to match it with a driver). * * This function registers a virtio device. This vdev's partent is * the rproc device. @@ -334,6 +335,13 @@ int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id) struct rproc_mem_entry *mem; int ret; + if (rproc->ops->kick == NULL) { + ret = -EINVAL; + dev_err(dev, ".kick method not defined for %s", + rproc->name); + goto out; + } + /* Try to find dedicated vdev buffer carveout */ mem = rproc_find_carveout_by_name(rproc, "vdev%dbuffer", rvdev->index); if (mem) { diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c index ee13d23b43a9..a6cbfa452764 100644 --- a/drivers/remoteproc/st_remoteproc.c +++ b/drivers/remoteproc/st_remoteproc.c @@ -190,7 +190,7 @@ static int st_rproc_start(struct rproc *rproc) } } - dev_info(&rproc->dev, "Started from 0x%x\n", rproc->bootaddr); + dev_info(&rproc->dev, "Started from 0x%llx\n", rproc->bootaddr); return 0; @@ -233,7 +233,7 @@ static const struct rproc_ops st_rproc_ops = { .parse_fw = st_rproc_parse_fw, .load = rproc_elf_load_segments, .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, - .sanity_check = rproc_elf_sanity_check, + .sanity_check = rproc_elf32_sanity_check, .get_boot_addr = rproc_elf_get_boot_addr, }; diff --git a/drivers/remoteproc/st_slim_rproc.c b/drivers/remoteproc/st_slim_rproc.c index 04492fead3c8..3cca8b65a8db 100644 --- a/drivers/remoteproc/st_slim_rproc.c +++ b/drivers/remoteproc/st_slim_rproc.c @@ -174,7 +174,7 @@ static int slim_rproc_stop(struct rproc *rproc) return 0; } -static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, int len) +static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct st_slim_rproc *slim_rproc = rproc->priv; void *va = NULL; @@ -191,7 +191,7 @@ static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, int len) } } - dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%x va = 0x%pK\n", + dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%zx va = 0x%pK\n", da, len, va); return va; @@ -203,7 +203,7 @@ static const struct rproc_ops slim_rproc_ops = { .da_to_va = slim_rproc_da_to_va, .get_boot_addr = rproc_elf_get_boot_addr, .load = rproc_elf_load_segments, - .sanity_check = rproc_elf_sanity_check, + .sanity_check = rproc_elf32_sanity_check, }; /** diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c index a18f88044111..6a66dbf2df40 100644 --- a/drivers/remoteproc/stm32_rproc.c +++ b/drivers/remoteproc/stm32_rproc.c @@ -505,7 +505,7 @@ static struct rproc_ops st_rproc_ops = { .load = rproc_elf_load_segments, .parse_fw = stm32_rproc_parse_fw, .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, - .sanity_check = rproc_elf_sanity_check, + .sanity_check = rproc_elf32_sanity_check, .get_boot_addr = rproc_elf_get_boot_addr, }; @@ -602,7 +602,7 @@ static int stm32_rproc_parse_dt(struct platform_device *pdev) err = stm32_rproc_get_syscon(np, "st,syscfg-pdds", &ddata->pdds); if (err) - dev_warn(dev, "failed to get pdds\n"); + dev_info(dev, "failed to get pdds\n"); rproc->auto_boot = of_property_read_bool(np, "st,auto-boot"); diff --git a/drivers/remoteproc/wkup_m3_rproc.c b/drivers/remoteproc/wkup_m3_rproc.c index 3984e585c847..b9349d684258 100644 --- a/drivers/remoteproc/wkup_m3_rproc.c +++ b/drivers/remoteproc/wkup_m3_rproc.c @@ -80,14 +80,14 @@ static int wkup_m3_rproc_stop(struct rproc *rproc) return 0; } -static void *wkup_m3_rproc_da_to_va(struct rproc *rproc, u64 da, int len) +static void *wkup_m3_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len) { struct wkup_m3_rproc *wkupm3 = rproc->priv; void *va = NULL; int i; u32 offset; - if (len <= 0) + if (len == 0) return NULL; for (i = 0; i < WKUPM3_MEM_MAX; i++) { diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c index 1c6c6a2e0def..440effed6df6 100644 --- a/drivers/soundwire/qcom.c +++ b/drivers/soundwire/qcom.c @@ -594,6 +594,7 @@ static int qcom_swrm_startup(struct snd_pcm_substream *substream, struct qcom_swrm_ctrl *ctrl = dev_get_drvdata(dai->dev); struct snd_soc_pcm_runtime *rtd = substream->private_data; struct sdw_stream_runtime *sruntime; + struct snd_soc_dai *codec_dai; int ret, i; sruntime = sdw_alloc_stream(dai->name); @@ -602,12 +603,12 @@ static int qcom_swrm_startup(struct snd_pcm_substream *substream, ctrl->sruntime[dai->id] = sruntime; - for (i = 0; i < rtd->num_codecs; i++) { - ret = snd_soc_dai_set_sdw_stream(rtd->codec_dais[i], sruntime, + for_each_rtd_codec_dais(rtd, i, codec_dai) { + ret = snd_soc_dai_set_sdw_stream(codec_dai, sruntime, substream->stream); if (ret < 0 && ret != -ENOTSUPP) { dev_err(dai->dev, "Failed to set sdw stream on %s", - rtd->codec_dais[i]->name); + codec_dai->name); sdw_release_stream(sruntime); return ret; } diff --git a/drivers/soundwire/stream.c b/drivers/soundwire/stream.c index 178ae92b8cc1..7fb89a94d9c0 100644 --- a/drivers/soundwire/stream.c +++ b/drivers/soundwire/stream.c @@ -167,13 +167,15 @@ static int sdw_program_slave_port_params(struct sdw_bus *bus, return ret; } - /* Program DPN_BlockCtrl1 register */ - ret = sdw_write(s_rt->slave, addr2, (p_params->bps - 1)); - if (ret < 0) { - dev_err(&s_rt->slave->dev, - "DPN_BlockCtrl1 register write failed for port %d\n", - t_params->port_num); - return ret; + if (!dpn_prop->read_only_wordlength) { + /* Program DPN_BlockCtrl1 register */ + ret = sdw_write(s_rt->slave, addr2, (p_params->bps - 1)); + if (ret < 0) { + dev_err(&s_rt->slave->dev, + "DPN_BlockCtrl1 register write failed for port %d\n", + t_params->port_num); + return ret; + } } /* Program DPN_SampleCtrl1 register */ diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index efce98e9844e..741b9140992a 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -575,7 +575,7 @@ config SPI_PPC4xx config SPI_PXA2XX tristate "PXA2xx SSP SPI master" - depends on (ARCH_PXA || ARCH_MMP || PCI || ACPI) + depends on ARCH_PXA || ARCH_MMP || PCI || ACPI || COMPILE_TEST select PXA_SSP if ARCH_PXA || ARCH_MMP help This enables using a PXA2xx or Sodaville SSP port as a SPI master diff --git a/fs/block_dev.c b/fs/block_dev.c index 9501880dff5e..52b6f646cdbd 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "internal.h" struct bdev_inode { @@ -2013,7 +2014,8 @@ ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) if (bdev_read_only(I_BDEV(bd_inode))) return -EPERM; - if (IS_SWAPFILE(bd_inode)) + /* uswsusp needs write permission to the swap */ + if (IS_SWAPFILE(bd_inode) && !hibernation_available()) return -ETXTBSY; if (!iov_iter_count(from)) diff --git a/fs/fs_parser.c b/fs/fs_parser.c index 7e6fb43f9541..ab53e42a874a 100644 --- a/fs/fs_parser.c +++ b/fs/fs_parser.c @@ -368,8 +368,6 @@ bool fs_validate_description(const char *name, const struct fs_parameter_spec *param, *p2; bool good = true; - pr_notice("*** VALIDATE %s ***\n", name); - for (param = desc; param->name; param++) { /* Check for duplicate parameter names */ for (p2 = desc; p2 < param; p2++) { diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index aff8642f0c2e..991c60c7ffe0 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -393,10 +393,9 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end) * In this case, we first scan the range and release found pages. * After releasing pages, hugetlb_unreserve_pages cleans up region/reserv * maps and global counts. Page faults can not race with truncation - * in this routine. hugetlb_no_page() prevents page faults in the - * truncated range. It checks i_size before allocation, and again after - * with the page table lock for the page held. The same lock must be - * acquired to unmap a page. + * in this routine. hugetlb_no_page() holds i_mmap_rwsem and prevents + * page faults in the truncated range by checking i_size. i_size is + * modified while holding i_mmap_rwsem. * hole punch is indicated if end is not LLONG_MAX * In the hole punch case we scan the range and release found pages. * Only when releasing a page is the associated region/reserv map @@ -436,7 +435,15 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart, index = page->index; hash = hugetlb_fault_mutex_hash(mapping, index); - mutex_lock(&hugetlb_fault_mutex_table[hash]); + if (!truncate_op) { + /* + * Only need to hold the fault mutex in the + * hole punch case. This prevents races with + * page faults. Races are not possible in the + * case of truncation. + */ + mutex_lock(&hugetlb_fault_mutex_table[hash]); + } /* * If page is mapped, it was faulted in after being @@ -450,7 +457,9 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart, if (unlikely(page_mapped(page))) { BUG_ON(truncate_op); + mutex_unlock(&hugetlb_fault_mutex_table[hash]); i_mmap_lock_write(mapping); + mutex_lock(&hugetlb_fault_mutex_table[hash]); hugetlb_vmdelete_list(&mapping->i_mmap, index * pages_per_huge_page(h), (index + 1) * pages_per_huge_page(h)); @@ -477,7 +486,8 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart, } unlock_page(page); - mutex_unlock(&hugetlb_fault_mutex_table[hash]); + if (!truncate_op) + mutex_unlock(&hugetlb_fault_mutex_table[hash]); } huge_pagevec_release(&pvec); cond_resched(); @@ -515,8 +525,8 @@ static int hugetlb_vmtruncate(struct inode *inode, loff_t offset) BUG_ON(offset & ~huge_page_mask(h)); pgoff = offset >> PAGE_SHIFT; - i_size_write(inode, offset); i_mmap_lock_write(mapping); + i_size_write(inode, offset); if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)) hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0); i_mmap_unlock_write(mapping); @@ -638,7 +648,11 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, /* addr is the offset within the file (zero based) */ addr = index * hpage_size; - /* mutex taken here, fault path and hole punch */ + /* + * fault mutex taken here, protects against fault path + * and hole punch. inode_lock previously taken protects + * against truncation. + */ hash = hugetlb_fault_mutex_hash(mapping, index); mutex_lock(&hugetlb_fault_mutex_table[hash]); diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 7c84c4c027c4..f080f542911b 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -503,7 +503,8 @@ EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate); int iomap_releasepage(struct page *page, gfp_t gfp_mask) { - trace_iomap_releasepage(page->mapping->host, page, 0, 0); + trace_iomap_releasepage(page->mapping->host, page_offset(page), + PAGE_SIZE); /* * mm accommodates an old ext3 case where clean pages might not have had @@ -520,7 +521,7 @@ EXPORT_SYMBOL_GPL(iomap_releasepage); void iomap_invalidatepage(struct page *page, unsigned int offset, unsigned int len) { - trace_iomap_invalidatepage(page->mapping->host, page, offset, len); + trace_iomap_invalidatepage(page->mapping->host, offset, len); /* * If we are invalidating the entire page, clear the dirty state from it @@ -1519,7 +1520,7 @@ iomap_do_writepage(struct page *page, struct writeback_control *wbc, void *data) u64 end_offset; loff_t offset; - trace_iomap_writepage(inode, page, 0, 0); + trace_iomap_writepage(inode, page_offset(page), PAGE_SIZE); /* * Refuse to write the page out if we are called from reclaim context. diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 631ac726b8cc..a326be6bfeeb 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -540,8 +540,8 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, /* * We are about to drop our additional submission reference, which - * might be the last reference to the dio. There are three three - * different ways we can progress here: + * might be the last reference to the dio. There are three different + * ways we can progress here: * * (a) If this is the last reference we will always complete and free * the dio ourselves. diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h index 6dc227b8c47e..4df19c66f597 100644 --- a/fs/iomap/trace.h +++ b/fs/iomap/trace.h @@ -41,14 +41,12 @@ DEFINE_EVENT(iomap_readpage_class, name, \ DEFINE_READPAGE_EVENT(iomap_readpage); DEFINE_READPAGE_EVENT(iomap_readpages); -DECLARE_EVENT_CLASS(iomap_page_class, - TP_PROTO(struct inode *inode, struct page *page, unsigned long off, - unsigned int len), - TP_ARGS(inode, page, off, len), +DECLARE_EVENT_CLASS(iomap_range_class, + TP_PROTO(struct inode *inode, unsigned long off, unsigned int len), + TP_ARGS(inode, off, len), TP_STRUCT__entry( __field(dev_t, dev) __field(u64, ino) - __field(pgoff_t, pgoff) __field(loff_t, size) __field(unsigned long, offset) __field(unsigned int, length) @@ -56,29 +54,26 @@ DECLARE_EVENT_CLASS(iomap_page_class, TP_fast_assign( __entry->dev = inode->i_sb->s_dev; __entry->ino = inode->i_ino; - __entry->pgoff = page_offset(page); __entry->size = i_size_read(inode); __entry->offset = off; __entry->length = len; ), - TP_printk("dev %d:%d ino 0x%llx pgoff 0x%lx size 0x%llx offset %lx " + TP_printk("dev %d:%d ino 0x%llx size 0x%llx offset %lx " "length %x", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, - __entry->pgoff, __entry->size, __entry->offset, __entry->length) ) -#define DEFINE_PAGE_EVENT(name) \ -DEFINE_EVENT(iomap_page_class, name, \ - TP_PROTO(struct inode *inode, struct page *page, unsigned long off, \ - unsigned int len), \ - TP_ARGS(inode, page, off, len)) -DEFINE_PAGE_EVENT(iomap_writepage); -DEFINE_PAGE_EVENT(iomap_releasepage); -DEFINE_PAGE_EVENT(iomap_invalidatepage); +#define DEFINE_RANGE_EVENT(name) \ +DEFINE_EVENT(iomap_range_class, name, \ + TP_PROTO(struct inode *inode, unsigned long off, unsigned int len),\ + TP_ARGS(inode, off, len)) +DEFINE_RANGE_EVENT(iomap_writepage); +DEFINE_RANGE_EVENT(iomap_releasepage); +DEFINE_RANGE_EVENT(iomap_invalidatepage); #define IOMAP_TYPE_STRINGS \ { IOMAP_HOLE, "HOLE" }, \ diff --git a/fs/ocfs2/alloc.c b/fs/ocfs2/alloc.c index 88534eb0e7c2..65b3abbcce4e 100644 --- a/fs/ocfs2/alloc.c +++ b/fs/ocfs2/alloc.c @@ -1060,7 +1060,6 @@ static int ocfs2_create_new_meta_bhs(handle_t *handle, brelse(bhs[i]); bhs[i] = NULL; } - mlog_errno(status); } return status; } @@ -3942,7 +3941,7 @@ static void ocfs2_insert_at_leaf(struct ocfs2_extent_tree *et, * above. * * This leaf needs to have space, either by the empty 1st - * extent record, or by virtue of an l_next_rec < l_count. + * extent record, or by virtue of an l_next_free_rec < l_count. */ ocfs2_rotate_leaf(el, insert_rec); } diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c index a368350d4c27..89d13e0705fe 100644 --- a/fs/ocfs2/cluster/heartbeat.c +++ b/fs/ocfs2/cluster/heartbeat.c @@ -101,8 +101,6 @@ static struct o2hb_callback { static struct o2hb_callback *hbcall_from_type(enum o2hb_callback_type type); -#define O2HB_DEFAULT_BLOCK_BITS 9 - enum o2hb_heartbeat_modes { O2HB_HEARTBEAT_LOCAL = 0, O2HB_HEARTBEAT_GLOBAL, @@ -1309,7 +1307,7 @@ static int o2hb_debug_open(struct inode *inode, struct file *file) case O2HB_DB_TYPE_REGION_NUMBER: reg = (struct o2hb_region *)db->db_data; - out += snprintf(buf + out, PAGE_SIZE - out, "%d\n", + out += scnprintf(buf + out, PAGE_SIZE - out, "%d\n", reg->hr_region_num); goto done; @@ -1319,12 +1317,12 @@ static int o2hb_debug_open(struct inode *inode, struct file *file) /* If 0, it has never been set before */ if (lts) lts = jiffies_to_msecs(jiffies - lts); - out += snprintf(buf + out, PAGE_SIZE - out, "%lu\n", lts); + out += scnprintf(buf + out, PAGE_SIZE - out, "%lu\n", lts); goto done; case O2HB_DB_TYPE_REGION_PINNED: reg = (struct o2hb_region *)db->db_data; - out += snprintf(buf + out, PAGE_SIZE - out, "%u\n", + out += scnprintf(buf + out, PAGE_SIZE - out, "%u\n", !!reg->hr_item_pinned); goto done; @@ -1333,8 +1331,8 @@ static int o2hb_debug_open(struct inode *inode, struct file *file) } while ((i = find_next_bit(map, db->db_len, i + 1)) < db->db_len) - out += snprintf(buf + out, PAGE_SIZE - out, "%d ", i); - out += snprintf(buf + out, PAGE_SIZE - out, "\n"); + out += scnprintf(buf + out, PAGE_SIZE - out, "%d ", i); + out += scnprintf(buf + out, PAGE_SIZE - out, "\n"); done: i_size_write(inode, out); diff --git a/fs/ocfs2/cluster/netdebug.c b/fs/ocfs2/cluster/netdebug.c index 02bf4a1774cc..667a5c5e1f66 100644 --- a/fs/ocfs2/cluster/netdebug.c +++ b/fs/ocfs2/cluster/netdebug.c @@ -443,8 +443,8 @@ static int o2net_fill_bitmap(char *buf, int len) o2net_fill_node_map(map, sizeof(map)); while ((i = find_next_bit(map, O2NM_MAX_NODES, i + 1)) < O2NM_MAX_NODES) - out += snprintf(buf + out, PAGE_SIZE - out, "%d ", i); - out += snprintf(buf + out, PAGE_SIZE - out, "\n"); + out += scnprintf(buf + out, PAGE_SIZE - out, "%d ", i); + out += scnprintf(buf + out, PAGE_SIZE - out, "\n"); return out; } diff --git a/fs/ocfs2/cluster/tcp.c b/fs/ocfs2/cluster/tcp.c index 48a3398f0bf5..2c512b40a940 100644 --- a/fs/ocfs2/cluster/tcp.c +++ b/fs/ocfs2/cluster/tcp.c @@ -1570,15 +1570,13 @@ static void o2net_start_connect(struct work_struct *work) struct sockaddr_in myaddr = {0, }, remoteaddr = {0, }; int ret = 0, stop; unsigned int timeout; - unsigned int noio_flag; + unsigned int nofs_flag; /* - * sock_create allocates the sock with GFP_KERNEL. We must set - * per-process flag PF_MEMALLOC_NOIO so that all allocations done - * by this process are done as if GFP_NOIO was specified. So we - * are not reentering filesystem while doing memory reclaim. + * sock_create allocates the sock with GFP_KERNEL. We must + * prevent the filesystem from being reentered by memory reclaim. */ - noio_flag = memalloc_noio_save(); + nofs_flag = memalloc_nofs_save(); /* if we're greater we initiate tx, otherwise we accept */ if (o2nm_this_node() <= o2net_num_from_nn(nn)) goto out; @@ -1683,7 +1681,7 @@ static void o2net_start_connect(struct work_struct *work) if (mynode) o2nm_node_put(mynode); - memalloc_noio_restore(noio_flag); + memalloc_nofs_restore(nofs_flag); return; } @@ -1810,15 +1808,13 @@ static int o2net_accept_one(struct socket *sock, int *more) struct o2nm_node *local_node = NULL; struct o2net_sock_container *sc = NULL; struct o2net_node *nn; - unsigned int noio_flag; + unsigned int nofs_flag; /* - * sock_create_lite allocates the sock with GFP_KERNEL. We must set - * per-process flag PF_MEMALLOC_NOIO so that all allocations done - * by this process are done as if GFP_NOIO was specified. So we - * are not reentering filesystem while doing memory reclaim. + * sock_create_lite allocates the sock with GFP_KERNEL. We must + * prevent the filesystem from being reentered by memory reclaim. */ - noio_flag = memalloc_noio_save(); + nofs_flag = memalloc_nofs_save(); BUG_ON(sock == NULL); *more = 0; @@ -1934,7 +1930,7 @@ static int o2net_accept_one(struct socket *sock, int *more) if (sc) sc_put(sc); - memalloc_noio_restore(noio_flag); + memalloc_nofs_restore(nofs_flag); return ret; } @@ -1948,7 +1944,6 @@ static void o2net_accept_many(struct work_struct *work) { struct socket *sock = o2net_listen_sock; int more; - int err; /* * It is critical to note that due to interrupt moderation @@ -1963,7 +1958,7 @@ static void o2net_accept_many(struct work_struct *work) */ for (;;) { - err = o2net_accept_one(sock, &more); + o2net_accept_one(sock, &more); if (!more) break; cond_resched(); diff --git a/fs/ocfs2/cluster/tcp.h b/fs/ocfs2/cluster/tcp.h index de87cbffd175..736338f45c59 100644 --- a/fs/ocfs2/cluster/tcp.h +++ b/fs/ocfs2/cluster/tcp.h @@ -32,7 +32,7 @@ struct o2net_msg __be32 status; __be32 key; __be32 msg_num; - __u8 buf[0]; + __u8 buf[]; }; typedef int (o2net_msg_handler_func)(struct o2net_msg *msg, u32 len, void *data, diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c index bdef72c0f099..5761060d2ba8 100644 --- a/fs/ocfs2/dir.c +++ b/fs/ocfs2/dir.c @@ -676,7 +676,7 @@ static struct buffer_head *ocfs2_find_entry_el(const char *name, int namelen, int ra_ptr = 0; /* Current index into readahead buffer */ int num = 0; - int nblocks, i, err; + int nblocks, i; sb = dir->i_sb; @@ -708,7 +708,7 @@ static struct buffer_head *ocfs2_find_entry_el(const char *name, int namelen, num++; bh = NULL; - err = ocfs2_read_dir_block(dir, b++, &bh, + ocfs2_read_dir_block(dir, b++, &bh, OCFS2_BH_READAHEAD); bh_use[ra_max] = bh; } diff --git a/fs/ocfs2/dlm/dlmcommon.h b/fs/ocfs2/dlm/dlmcommon.h index 0463dce65bb2..c8a444622faa 100644 --- a/fs/ocfs2/dlm/dlmcommon.h +++ b/fs/ocfs2/dlm/dlmcommon.h @@ -564,7 +564,7 @@ struct dlm_migratable_lockres // 48 bytes u8 lvb[DLM_LVB_LEN]; // 112 bytes - struct dlm_migratable_lock ml[0]; // 16 bytes each, begins at byte 112 + struct dlm_migratable_lock ml[]; // 16 bytes each, begins at byte 112 }; #define DLM_MIG_LOCKRES_MAX_LEN \ (sizeof(struct dlm_migratable_lockres) + \ @@ -601,7 +601,7 @@ struct dlm_convert_lock u8 name[O2NM_MAX_NAME_LEN]; - s8 lvb[0]; + s8 lvb[]; }; #define DLM_CONVERT_LOCK_MAX_LEN (sizeof(struct dlm_convert_lock)+DLM_LVB_LEN) @@ -616,7 +616,7 @@ struct dlm_unlock_lock u8 name[O2NM_MAX_NAME_LEN]; - s8 lvb[0]; + s8 lvb[]; }; #define DLM_UNLOCK_LOCK_MAX_LEN (sizeof(struct dlm_unlock_lock)+DLM_LVB_LEN) @@ -632,7 +632,7 @@ struct dlm_proxy_ast u8 name[O2NM_MAX_NAME_LEN]; - s8 lvb[0]; + s8 lvb[]; }; #define DLM_PROXY_AST_MAX_LEN (sizeof(struct dlm_proxy_ast)+DLM_LVB_LEN) diff --git a/fs/ocfs2/dlm/dlmdebug.c b/fs/ocfs2/dlm/dlmdebug.c index c5c6efba7b5e..4b8b41d23e91 100644 --- a/fs/ocfs2/dlm/dlmdebug.c +++ b/fs/ocfs2/dlm/dlmdebug.c @@ -244,11 +244,11 @@ static int stringify_lockname(const char *lockname, int locklen, char *buf, memcpy((__be64 *)&inode_blkno_be, (char *)&lockname[OCFS2_DENTRY_LOCK_INO_START], sizeof(__be64)); - out += snprintf(buf + out, len - out, "%.*s%08x", + out += scnprintf(buf + out, len - out, "%.*s%08x", OCFS2_DENTRY_LOCK_INO_START - 1, lockname, (unsigned int)be64_to_cpu(inode_blkno_be)); } else - out += snprintf(buf + out, len - out, "%.*s", + out += scnprintf(buf + out, len - out, "%.*s", locklen, lockname); return out; } @@ -260,7 +260,7 @@ static int stringify_nodemap(unsigned long *nodemap, int maxnodes, int i = -1; while ((i = find_next_bit(nodemap, maxnodes, i + 1)) < maxnodes) - out += snprintf(buf + out, len - out, "%d ", i); + out += scnprintf(buf + out, len - out, "%d ", i); return out; } @@ -278,34 +278,34 @@ static int dump_mle(struct dlm_master_list_entry *mle, char *buf, int len) mle_type = "MIG"; out += stringify_lockname(mle->mname, mle->mnamelen, buf + out, len - out); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "\t%3s\tmas=%3u\tnew=%3u\tevt=%1d\tuse=%1d\tref=%3d\n", mle_type, mle->master, mle->new_master, !list_empty(&mle->hb_events), !!mle->inuse, kref_read(&mle->mle_refs)); - out += snprintf(buf + out, len - out, "Maybe="); + out += scnprintf(buf + out, len - out, "Maybe="); out += stringify_nodemap(mle->maybe_map, O2NM_MAX_NODES, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); - out += snprintf(buf + out, len - out, "Vote="); + out += scnprintf(buf + out, len - out, "Vote="); out += stringify_nodemap(mle->vote_map, O2NM_MAX_NODES, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); - out += snprintf(buf + out, len - out, "Response="); + out += scnprintf(buf + out, len - out, "Response="); out += stringify_nodemap(mle->response_map, O2NM_MAX_NODES, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); - out += snprintf(buf + out, len - out, "Node="); + out += scnprintf(buf + out, len - out, "Node="); out += stringify_nodemap(mle->node_map, O2NM_MAX_NODES, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); return out; } @@ -353,7 +353,7 @@ static int debug_purgelist_print(struct dlm_ctxt *dlm, char *buf, int len) int out = 0; unsigned long total = 0; - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Dumping Purgelist for Domain: %s\n", dlm->name); spin_lock(&dlm->spinlock); @@ -365,13 +365,13 @@ static int debug_purgelist_print(struct dlm_ctxt *dlm, char *buf, int len) out += stringify_lockname(res->lockname.name, res->lockname.len, buf + out, len - out); - out += snprintf(buf + out, len - out, "\t%ld\n", + out += scnprintf(buf + out, len - out, "\t%ld\n", (jiffies - res->last_used)/HZ); spin_unlock(&res->spinlock); } spin_unlock(&dlm->spinlock); - out += snprintf(buf + out, len - out, "Total on list: %lu\n", total); + out += scnprintf(buf + out, len - out, "Total on list: %lu\n", total); return out; } @@ -410,7 +410,7 @@ static int debug_mle_print(struct dlm_ctxt *dlm, char *buf, int len) int i, out = 0; unsigned long total = 0, longest = 0, bucket_count = 0; - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Dumping MLEs for Domain: %s\n", dlm->name); spin_lock(&dlm->master_lock); @@ -428,7 +428,7 @@ static int debug_mle_print(struct dlm_ctxt *dlm, char *buf, int len) } spin_unlock(&dlm->master_lock); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Total: %lu, Longest: %lu\n", total, longest); return out; } @@ -467,7 +467,7 @@ static int dump_lock(struct dlm_lock *lock, int list_type, char *buf, int len) #define DEBUG_LOCK_VERSION 1 spin_lock(&lock->spinlock); - out = snprintf(buf, len, "LOCK:%d,%d,%d,%d,%d,%d:%lld,%d,%d,%d,%d,%d," + out = scnprintf(buf, len, "LOCK:%d,%d,%d,%d,%d,%d:%lld,%d,%d,%d,%d,%d," "%d,%d,%d,%d\n", DEBUG_LOCK_VERSION, list_type, lock->ml.type, lock->ml.convert_type, @@ -491,13 +491,13 @@ static int dump_lockres(struct dlm_lock_resource *res, char *buf, int len) int i; int out = 0; - out += snprintf(buf + out, len - out, "NAME:"); + out += scnprintf(buf + out, len - out, "NAME:"); out += stringify_lockname(res->lockname.name, res->lockname.len, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); #define DEBUG_LRES_VERSION 1 - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "LRES:%d,%d,%d,%ld,%d,%d,%d,%d,%d,%d,%d\n", DEBUG_LRES_VERSION, res->owner, res->state, res->last_used, @@ -509,17 +509,17 @@ static int dump_lockres(struct dlm_lock_resource *res, char *buf, int len) kref_read(&res->refs)); /* refmap */ - out += snprintf(buf + out, len - out, "RMAP:"); + out += scnprintf(buf + out, len - out, "RMAP:"); out += stringify_nodemap(res->refmap, O2NM_MAX_NODES, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); /* lvb */ - out += snprintf(buf + out, len - out, "LVBX:"); + out += scnprintf(buf + out, len - out, "LVBX:"); for (i = 0; i < DLM_LVB_LEN; i++) - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%02x", (unsigned char)res->lvb[i]); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); /* granted */ list_for_each_entry(lock, &res->granted, list) @@ -533,7 +533,7 @@ static int dump_lockres(struct dlm_lock_resource *res, char *buf, int len) list_for_each_entry(lock, &res->blocked, list) out += dump_lock(lock, 2, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); return out; } @@ -683,41 +683,41 @@ static int debug_state_print(struct dlm_ctxt *dlm, char *buf, int len) } /* Domain: xxxxxxxxxx Key: 0xdfbac769 */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Domain: %s Key: 0x%08x Protocol: %d.%d\n", dlm->name, dlm->key, dlm->dlm_locking_proto.pv_major, dlm->dlm_locking_proto.pv_minor); /* Thread Pid: xxx Node: xxx State: xxxxx */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Thread Pid: %d Node: %d State: %s\n", task_pid_nr(dlm->dlm_thread_task), dlm->node_num, state); /* Number of Joins: xxx Joining Node: xxx */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Number of Joins: %d Joining Node: %d\n", dlm->num_joins, dlm->joining_node); /* Domain Map: xx xx xx */ - out += snprintf(buf + out, len - out, "Domain Map: "); + out += scnprintf(buf + out, len - out, "Domain Map: "); out += stringify_nodemap(dlm->domain_map, O2NM_MAX_NODES, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); /* Exit Domain Map: xx xx xx */ - out += snprintf(buf + out, len - out, "Exit Domain Map: "); + out += scnprintf(buf + out, len - out, "Exit Domain Map: "); out += stringify_nodemap(dlm->exit_domain_map, O2NM_MAX_NODES, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); /* Live Map: xx xx xx */ - out += snprintf(buf + out, len - out, "Live Map: "); + out += scnprintf(buf + out, len - out, "Live Map: "); out += stringify_nodemap(dlm->live_nodes_map, O2NM_MAX_NODES, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); /* Lock Resources: xxx (xxx) */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Lock Resources: %d (%d)\n", atomic_read(&dlm->res_cur_count), atomic_read(&dlm->res_tot_count)); @@ -729,29 +729,29 @@ static int debug_state_print(struct dlm_ctxt *dlm, char *buf, int len) cur_mles += atomic_read(&dlm->mle_cur_count[i]); /* MLEs: xxx (xxx) */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "MLEs: %d (%d)\n", cur_mles, tot_mles); /* Blocking: xxx (xxx) */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, " Blocking: %d (%d)\n", atomic_read(&dlm->mle_cur_count[DLM_MLE_BLOCK]), atomic_read(&dlm->mle_tot_count[DLM_MLE_BLOCK])); /* Mastery: xxx (xxx) */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, " Mastery: %d (%d)\n", atomic_read(&dlm->mle_cur_count[DLM_MLE_MASTER]), atomic_read(&dlm->mle_tot_count[DLM_MLE_MASTER])); /* Migration: xxx (xxx) */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, " Migration: %d (%d)\n", atomic_read(&dlm->mle_cur_count[DLM_MLE_MIGRATION]), atomic_read(&dlm->mle_tot_count[DLM_MLE_MIGRATION])); /* Lists: Dirty=Empty Purge=InUse PendingASTs=Empty ... */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Lists: Dirty=%s Purge=%s PendingASTs=%s " "PendingBASTs=%s\n", (list_empty(&dlm->dirty_list) ? "Empty" : "InUse"), @@ -760,12 +760,12 @@ static int debug_state_print(struct dlm_ctxt *dlm, char *buf, int len) (list_empty(&dlm->pending_basts) ? "Empty" : "InUse")); /* Purge Count: xxx Refs: xxx */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Purge Count: %d Refs: %d\n", dlm->purge_count, kref_read(&dlm->dlm_refs)); /* Dead Node: xxx */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Dead Node: %d\n", dlm->reco.dead_node); /* What about DLM_RECO_STATE_FINALIZE? */ @@ -775,19 +775,19 @@ static int debug_state_print(struct dlm_ctxt *dlm, char *buf, int len) state = "INACTIVE"; /* Recovery Pid: xxxx Master: xxx State: xxxx */ - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "Recovery Pid: %d Master: %d State: %s\n", task_pid_nr(dlm->dlm_reco_thread_task), dlm->reco.new_master, state); /* Recovery Map: xx xx */ - out += snprintf(buf + out, len - out, "Recovery Map: "); + out += scnprintf(buf + out, len - out, "Recovery Map: "); out += stringify_nodemap(dlm->recovery_map, O2NM_MAX_NODES, buf + out, len - out); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); /* Recovery Node State: */ - out += snprintf(buf + out, len - out, "Recovery Node State:\n"); + out += scnprintf(buf + out, len - out, "Recovery Node State:\n"); list_for_each_entry(node, &dlm->reco.node_data, list) { switch (node->state) { case DLM_RECO_NODE_DATA_INIT: @@ -815,7 +815,7 @@ static int debug_state_print(struct dlm_ctxt *dlm, char *buf, int len) state = "BAD"; break; } - out += snprintf(buf + out, len - out, "\t%u - %s\n", + out += scnprintf(buf + out, len - out, "\t%u - %s\n", node->node_num, state); } diff --git a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c index 900f7e466d11..55a6512e9fde 100644 --- a/fs/ocfs2/dlm/dlmmaster.c +++ b/fs/ocfs2/dlm/dlmmaster.c @@ -2749,8 +2749,6 @@ static int dlm_migrate_lockres(struct dlm_ctxt *dlm, return ret; } -#define DLM_MIGRATION_RETRY_MS 100 - /* * Should be called only after beginning the domain leave process. * There should not be any remaining locks on nonlocal lock resources, diff --git a/fs/ocfs2/dlm/dlmthread.c b/fs/ocfs2/dlm/dlmthread.c index fd40c17cd022..5ccc4ff0b82a 100644 --- a/fs/ocfs2/dlm/dlmthread.c +++ b/fs/ocfs2/dlm/dlmthread.c @@ -39,8 +39,6 @@ static int dlm_thread(void *data); static void dlm_flush_asts(struct dlm_ctxt *dlm); -#define dlm_lock_is_remote(dlm, lock) ((lock)->ml.node != (dlm)->node_num) - /* will exit holding res->spinlock, but may drop in function */ /* waits until flags are cleared on res->state */ void __dlm_wait_on_lockres_flags(struct dlm_lock_resource *res, int flags) @@ -680,7 +678,6 @@ static void dlm_flush_asts(struct dlm_ctxt *dlm) #define DLM_THREAD_TIMEOUT_MS (4 * 1000) #define DLM_THREAD_MAX_DIRTY 100 -#define DLM_THREAD_MAX_ASTS 10 static int dlm_thread(void *data) { diff --git a/fs/ocfs2/dlmglue.c b/fs/ocfs2/dlmglue.c index cb9e6a73bea9..152a0fc4e905 100644 --- a/fs/ocfs2/dlmglue.c +++ b/fs/ocfs2/dlmglue.c @@ -2133,7 +2133,7 @@ static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, } #define OCFS2_SEC_BITS 34 -#define OCFS2_SEC_SHIFT (64 - 34) +#define OCFS2_SEC_SHIFT (64 - OCFS2_SEC_BITS) #define OCFS2_NSEC_MASK ((1ULL << OCFS2_SEC_SHIFT) - 1) /* LVB only has room for 64 bits of time here so we pack it for diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c index 68ba354cf361..b425f0b01dce 100644 --- a/fs/ocfs2/journal.c +++ b/fs/ocfs2/journal.c @@ -91,7 +91,7 @@ enum ocfs2_replay_state { struct ocfs2_replay_map { unsigned int rm_slots; enum ocfs2_replay_state rm_state; - unsigned char rm_replay_slots[0]; + unsigned char rm_replay_slots[]; }; static void ocfs2_replay_map_set_state(struct ocfs2_super *osb, int state) diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index da65251ef815..5381020aaa9a 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -406,7 +406,7 @@ static int ocfs2_mknod(struct inode *dir, if (status < 0) { mlog_errno(status); - goto leave; + goto roll_back; } if (si.enable) { @@ -414,7 +414,7 @@ static int ocfs2_mknod(struct inode *dir, meta_ac, data_ac); if (status < 0) { mlog_errno(status); - goto leave; + goto roll_back; } } @@ -427,7 +427,7 @@ static int ocfs2_mknod(struct inode *dir, OCFS2_I(dir)->ip_blkno); if (status) { mlog_errno(status); - goto leave; + goto roll_back; } dl = dentry->d_fsdata; @@ -437,12 +437,19 @@ static int ocfs2_mknod(struct inode *dir, &lookup); if (status < 0) { mlog_errno(status); - goto leave; + goto roll_back; } insert_inode_hash(inode); d_instantiate(dentry, inode); status = 0; + +roll_back: + if (status < 0 && S_ISDIR(mode)) { + ocfs2_add_links_count(dirfe, -1); + drop_nlink(dir); + } + leave: if (status < 0 && did_quota_inode) dquot_free_inode(inode); diff --git a/fs/ocfs2/ocfs2_fs.h b/fs/ocfs2/ocfs2_fs.h index 0db4a7ec58a2..0dd8c41bafd4 100644 --- a/fs/ocfs2/ocfs2_fs.h +++ b/fs/ocfs2/ocfs2_fs.h @@ -470,7 +470,7 @@ struct ocfs2_extent_list { __le16 l_reserved1; __le64 l_reserved2; /* Pad to sizeof(ocfs2_extent_rec) */ -/*10*/ struct ocfs2_extent_rec l_recs[0]; /* Extent records */ +/*10*/ struct ocfs2_extent_rec l_recs[]; /* Extent records */ }; /* @@ -484,7 +484,7 @@ struct ocfs2_chain_list { __le16 cl_count; /* Total chains in this list */ __le16 cl_next_free_rec; /* Next unused chain slot */ __le64 cl_reserved1; -/*10*/ struct ocfs2_chain_rec cl_recs[0]; /* Chain records */ +/*10*/ struct ocfs2_chain_rec cl_recs[]; /* Chain records */ }; /* @@ -496,7 +496,7 @@ struct ocfs2_truncate_log { /*00*/ __le16 tl_count; /* Total records in this log */ __le16 tl_used; /* Number of records in use */ __le32 tl_reserved1; -/*08*/ struct ocfs2_truncate_rec tl_recs[0]; /* Truncate records */ +/*08*/ struct ocfs2_truncate_rec tl_recs[]; /* Truncate records */ }; /* @@ -640,7 +640,7 @@ struct ocfs2_local_alloc __le16 la_size; /* Size of included bitmap, in bytes */ __le16 la_reserved1; __le64 la_reserved2; -/*10*/ __u8 la_bitmap[0]; +/*10*/ __u8 la_bitmap[]; }; /* @@ -653,7 +653,7 @@ struct ocfs2_inline_data * for data, starting at id_data */ __le16 id_reserved0; __le32 id_reserved1; - __u8 id_data[0]; /* Start of user data */ + __u8 id_data[]; /* Start of user data */ }; /* @@ -798,7 +798,7 @@ struct ocfs2_dx_entry_list { * possible in de_entries */ __le16 de_num_used; /* Current number of * de_entries entries */ - struct ocfs2_dx_entry de_entries[0]; /* Indexed dir entries + struct ocfs2_dx_entry de_entries[]; /* Indexed dir entries * in a packed array of * length de_num_used */ }; @@ -935,7 +935,7 @@ struct ocfs2_refcount_list { __le16 rl_used; /* Current number of used records */ __le32 rl_reserved2; __le64 rl_reserved1; /* Pad to sizeof(ocfs2_refcount_record) */ -/*10*/ struct ocfs2_refcount_rec rl_recs[0]; /* Refcount records */ +/*10*/ struct ocfs2_refcount_rec rl_recs[]; /* Refcount records */ }; @@ -1021,7 +1021,7 @@ struct ocfs2_xattr_header { buckets. A block uses xb_check and sets this field to zero.) */ - struct ocfs2_xattr_entry xh_entries[0]; /* xattr entry list. */ + struct ocfs2_xattr_entry xh_entries[]; /* xattr entry list. */ }; /* @@ -1207,7 +1207,7 @@ struct ocfs2_local_disk_dqinfo { /* Header of one chunk of a quota file */ struct ocfs2_local_disk_chunk { __le32 dqc_free; /* Number of free entries in the bitmap */ - __u8 dqc_bitmap[0]; /* Bitmap of entries in the corresponding + __u8 dqc_bitmap[]; /* Bitmap of entries in the corresponding * chunk of quota file */ }; diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index ee43e51188be..cfb77f70c888 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -154,6 +154,7 @@ ocfs2_refcount_cache_get_super(struct ocfs2_caching_info *ci) } static void ocfs2_refcount_cache_lock(struct ocfs2_caching_info *ci) +__acquires(&rf->rf_lock) { struct ocfs2_refcount_tree *rf = cache_info_to_refcount(ci); @@ -161,6 +162,7 @@ static void ocfs2_refcount_cache_lock(struct ocfs2_caching_info *ci) } static void ocfs2_refcount_cache_unlock(struct ocfs2_caching_info *ci) +__releases(&rf->rf_lock) { struct ocfs2_refcount_tree *rf = cache_info_to_refcount(ci); diff --git a/fs/ocfs2/reservations.c b/fs/ocfs2/reservations.c index 0249e8ca1028..bf3842e34fb9 100644 --- a/fs/ocfs2/reservations.c +++ b/fs/ocfs2/reservations.c @@ -33,9 +33,6 @@ static DEFINE_SPINLOCK(resv_lock); -#define OCFS2_MIN_RESV_WINDOW_BITS 8 -#define OCFS2_MAX_RESV_WINDOW_BITS 1024 - int ocfs2_dir_resv_allowed(struct ocfs2_super *osb) { return (osb->osb_resv_level && osb->osb_dir_resv_level); diff --git a/fs/ocfs2/stackglue.c b/fs/ocfs2/stackglue.c index 8aa6a667860c..a191094694c6 100644 --- a/fs/ocfs2/stackglue.c +++ b/fs/ocfs2/stackglue.c @@ -656,8 +656,6 @@ static int ocfs2_sysfs_init(void) * and easier to preserve the name. */ -#define FS_OCFS2_NM 1 - static struct ctl_table ocfs2_nm_table[] = { { .procname = "hb_ctl_path", diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c index 939df99d2dec..4836becb7578 100644 --- a/fs/ocfs2/suballoc.c +++ b/fs/ocfs2/suballoc.c @@ -2509,9 +2509,6 @@ static int _ocfs2_free_suballoc_bits(handle_t *handle, bail: brelse(group_bh); - - if (status) - mlog_errno(status); return status; } @@ -2582,8 +2579,6 @@ static int _ocfs2_free_clusters(handle_t *handle, num_clusters); out: - if (status) - mlog_errno(status); return status; } diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index 05dd68ade293..ac61eeaf3837 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -220,31 +220,31 @@ static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len) int i, out = 0; unsigned long flags; - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => Id: %-s Uuid: %-s Gen: 0x%X Label: %-s\n", "Device", osb->dev_str, osb->uuid_str, osb->fs_generation, osb->vol_label); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => State: %d Flags: 0x%lX\n", "Volume", atomic_read(&osb->vol_state), osb->osb_flags); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => Block: %lu Cluster: %d\n", "Sizes", osb->sb->s_blocksize, osb->s_clustersize); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => Compat: 0x%X Incompat: 0x%X " "ROcompat: 0x%X\n", "Features", osb->s_feature_compat, osb->s_feature_incompat, osb->s_feature_ro_compat); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => Opts: 0x%lX AtimeQuanta: %u\n", "Mount", osb->s_mount_opt, osb->s_atime_quantum); if (cconn) { - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => Stack: %s Name: %*s " "Version: %d.%d\n", "Cluster", (*osb->osb_cluster_stack == '\0' ? @@ -255,7 +255,7 @@ static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len) } spin_lock_irqsave(&osb->dc_task_lock, flags); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => Pid: %d Count: %lu WakeSeq: %lu " "WorkSeq: %lu\n", "DownCnvt", (osb->dc_task ? task_pid_nr(osb->dc_task) : -1), @@ -264,32 +264,32 @@ static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len) spin_unlock_irqrestore(&osb->dc_task_lock, flags); spin_lock(&osb->osb_lock); - out += snprintf(buf + out, len - out, "%10s => Pid: %d Nodes:", + out += scnprintf(buf + out, len - out, "%10s => Pid: %d Nodes:", "Recovery", (osb->recovery_thread_task ? task_pid_nr(osb->recovery_thread_task) : -1)); if (rm->rm_used == 0) - out += snprintf(buf + out, len - out, " None\n"); + out += scnprintf(buf + out, len - out, " None\n"); else { for (i = 0; i < rm->rm_used; i++) - out += snprintf(buf + out, len - out, " %d", + out += scnprintf(buf + out, len - out, " %d", rm->rm_entries[i]); - out += snprintf(buf + out, len - out, "\n"); + out += scnprintf(buf + out, len - out, "\n"); } spin_unlock(&osb->osb_lock); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => Pid: %d Interval: %lu\n", "Commit", (osb->commit_task ? task_pid_nr(osb->commit_task) : -1), osb->osb_commit_interval); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => State: %d TxnId: %lu NumTxns: %d\n", "Journal", osb->journal->j_state, osb->journal->j_trans_id, atomic_read(&osb->journal->j_num_trans)); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => GlobalAllocs: %d LocalAllocs: %d " "SubAllocs: %d LAWinMoves: %d SAExtends: %d\n", "Stats", @@ -299,7 +299,7 @@ static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len) atomic_read(&osb->alloc_stats.moves), atomic_read(&osb->alloc_stats.bg_extends)); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => State: %u Descriptor: %llu Size: %u bits " "Default: %u bits\n", "LocalAlloc", osb->local_alloc_state, @@ -307,7 +307,7 @@ static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len) osb->local_alloc_bits, osb->local_alloc_default_bits); spin_lock(&osb->osb_lock); - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s => InodeSlot: %d StolenInodes: %d, " "MetaSlot: %d StolenMeta: %d\n", "Steal", osb->s_inode_steal_slot, @@ -316,20 +316,20 @@ static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len) atomic_read(&osb->s_num_meta_stolen)); spin_unlock(&osb->osb_lock); - out += snprintf(buf + out, len - out, "OrphanScan => "); - out += snprintf(buf + out, len - out, "Local: %u Global: %u ", + out += scnprintf(buf + out, len - out, "OrphanScan => "); + out += scnprintf(buf + out, len - out, "Local: %u Global: %u ", os->os_count, os->os_seqno); - out += snprintf(buf + out, len - out, " Last Scan: "); + out += scnprintf(buf + out, len - out, " Last Scan: "); if (atomic_read(&os->os_state) == ORPHAN_SCAN_INACTIVE) - out += snprintf(buf + out, len - out, "Disabled\n"); + out += scnprintf(buf + out, len - out, "Disabled\n"); else - out += snprintf(buf + out, len - out, "%lu seconds ago\n", + out += scnprintf(buf + out, len - out, "%lu seconds ago\n", (unsigned long)(ktime_get_seconds() - os->os_scantime)); - out += snprintf(buf + out, len - out, "%10s => %3s %10s\n", + out += scnprintf(buf + out, len - out, "%10s => %3s %10s\n", "Slots", "Num", "RecoGen"); for (i = 0; i < osb->max_slots; ++i) { - out += snprintf(buf + out, len - out, + out += scnprintf(buf + out, len - out, "%10s %c %3d %10d\n", " ", (i == osb->slot_num ? '*' : ' '), diff --git a/fs/pipe.c b/fs/pipe.c index 2144507447c5..16fb72e9abf7 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -146,7 +146,7 @@ static int anon_pipe_buf_steal(struct pipe_inode_info *pipe, struct page *page = buf->page; if (page_count(page) == 1) { - memcg_kmem_uncharge(page, 0); + memcg_kmem_uncharge_page(page, 0); __SetPageLocked(page); return 0; } diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index ffda9d514239..3c4823a32e62 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -334,6 +334,30 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, return ret; } +/* Should pair with userfaultfd_signal_pending() */ +static inline long userfaultfd_get_blocking_state(unsigned int flags) +{ + if (flags & FAULT_FLAG_INTERRUPTIBLE) + return TASK_INTERRUPTIBLE; + + if (flags & FAULT_FLAG_KILLABLE) + return TASK_KILLABLE; + + return TASK_UNINTERRUPTIBLE; +} + +/* Should pair with userfaultfd_get_blocking_state() */ +static inline bool userfaultfd_signal_pending(unsigned int flags) +{ + if (flags & FAULT_FLAG_INTERRUPTIBLE) + return signal_pending(current); + + if (flags & FAULT_FLAG_KILLABLE) + return fatal_signal_pending(current); + + return false; +} + /* * The locking rules involved in returning VM_FAULT_RETRY depending on * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and @@ -355,7 +379,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) struct userfaultfd_ctx *ctx; struct userfaultfd_wait_queue uwq; vm_fault_t ret = VM_FAULT_SIGBUS; - bool must_wait, return_to_userland; + bool must_wait; long blocking_state; /* @@ -462,11 +486,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) uwq.ctx = ctx; uwq.waken = false; - return_to_userland = - (vmf->flags & (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE)) == - (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE); - blocking_state = return_to_userland ? TASK_INTERRUPTIBLE : - TASK_KILLABLE; + blocking_state = userfaultfd_get_blocking_state(vmf->flags); spin_lock_irq(&ctx->fault_pending_wqh.lock); /* @@ -492,8 +512,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) up_read(&mm->mmap_sem); if (likely(must_wait && !READ_ONCE(ctx->released) && - (return_to_userland ? !signal_pending(current) : - !fatal_signal_pending(current)))) { + !userfaultfd_signal_pending(vmf->flags))) { wake_up_poll(&ctx->fd_wqh, EPOLLIN); schedule(); ret |= VM_FAULT_MAJOR; @@ -515,8 +534,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) set_current_state(blocking_state); if (READ_ONCE(uwq.waken) || READ_ONCE(ctx->released) || - (return_to_userland ? signal_pending(current) : - fatal_signal_pending(current))) + userfaultfd_signal_pending(vmf->flags)) break; schedule(); } @@ -524,30 +542,6 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) __set_current_state(TASK_RUNNING); - if (return_to_userland) { - if (signal_pending(current) && - !fatal_signal_pending(current)) { - /* - * If we got a SIGSTOP or SIGCONT and this is - * a normal userland page fault, just let - * userland return so the signal will be - * handled and gdb debugging works. The page - * fault code immediately after we return from - * this function is going to release the - * mmap_sem and it's not depending on it - * (unlike gup would if we were not to return - * VM_FAULT_RETRY). - * - * If a fatal signal is pending we still take - * the streamlined VM_FAULT_RETRY failure path - * and there's no need to retake the mmap_sem - * in such case. - */ - down_read(&mm->mmap_sem); - ret = VM_FAULT_NOPAGE; - } - } - /* * Here we race with the list_del; list_add in * userfaultfd_ctx_read(), however because we don't ever run diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index aceca2f9a3db..4f95df476181 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -26,6 +26,7 @@ xfs-y += $(addprefix libxfs/, \ xfs_bmap.o \ xfs_bmap_btree.o \ xfs_btree.o \ + xfs_btree_staging.o \ xfs_da_btree.o \ xfs_defer.o \ xfs_dir2.o \ diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c index 08d6beb54f8c..9d84007a5c65 100644 --- a/fs/xfs/libxfs/xfs_ag.c +++ b/fs/xfs/libxfs/xfs_ag.c @@ -231,7 +231,7 @@ xfs_sbblock_init( struct xfs_buf *bp, struct aghdr_init_data *id) { - struct xfs_dsb *dsb = XFS_BUF_TO_SBP(bp); + struct xfs_dsb *dsb = bp->b_addr; xfs_sb_to_disk(dsb, &mp->m_sb); dsb->sb_inprogress = 1; @@ -243,7 +243,7 @@ xfs_agfblock_init( struct xfs_buf *bp, struct aghdr_init_data *id) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(bp); + struct xfs_agf *agf = bp->b_addr; xfs_extlen_t tmpsize; agf->agf_magicnum = cpu_to_be32(XFS_AGF_MAGIC); @@ -301,7 +301,7 @@ xfs_agflblock_init( uuid_copy(&agfl->agfl_uuid, &mp->m_sb.sb_meta_uuid); } - agfl_bno = XFS_BUF_TO_AGFL_BNO(mp, bp); + agfl_bno = xfs_buf_to_agfl_bno(bp); for (bucket = 0; bucket < xfs_agfl_size(mp); bucket++) agfl_bno[bucket] = cpu_to_be32(NULLAGBLOCK); } @@ -312,7 +312,7 @@ xfs_agiblock_init( struct xfs_buf *bp, struct aghdr_init_data *id) { - struct xfs_agi *agi = XFS_BUF_TO_AGI(bp); + struct xfs_agi *agi = bp->b_addr; int bucket; agi->agi_magicnum = cpu_to_be32(XFS_AGI_MAGIC); @@ -502,7 +502,7 @@ xfs_ag_extend_space( if (error) return error; - agi = XFS_BUF_TO_AGI(bp); + agi = bp->b_addr; be32_add_cpu(&agi->agi_length, len); ASSERT(id->agno == mp->m_sb.sb_agcount - 1 || be32_to_cpu(agi->agi_length) == mp->m_sb.sb_agblocks); @@ -515,7 +515,7 @@ xfs_ag_extend_space( if (error) return error; - agf = XFS_BUF_TO_AGF(bp); + agf = bp->b_addr; be32_add_cpu(&agf->agf_length, len); ASSERT(agf->agf_length == agi->agi_length); xfs_alloc_log_agf(tp, bp, XFS_AGF_LENGTH); @@ -569,11 +569,11 @@ xfs_ag_get_geometry( memset(ageo, 0, sizeof(*ageo)); ageo->ag_number = agno; - agi = XFS_BUF_TO_AGI(agi_bp); + agi = agi_bp->b_addr; ageo->ag_icount = be32_to_cpu(agi->agi_count); ageo->ag_ifree = be32_to_cpu(agi->agi_freecount); - agf = XFS_BUF_TO_AGF(agf_bp); + agf = agf_bp->b_addr; ageo->ag_length = be32_to_cpu(agf->agf_length); freeblks = pag->pagf_freeblks + pag->pagf_flcount + diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c index d8053bc96c4d..203e74fa64aa 100644 --- a/fs/xfs/libxfs/xfs_alloc.c +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -151,7 +151,7 @@ xfs_alloc_lookup_eq( cur->bc_rec.a.ar_startblock = bno; cur->bc_rec.a.ar_blockcount = len; error = xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat); - cur->bc_private.a.priv.abt.active = (*stat == 1); + cur->bc_ag.abt.active = (*stat == 1); return error; } @@ -171,7 +171,7 @@ xfs_alloc_lookup_ge( cur->bc_rec.a.ar_startblock = bno; cur->bc_rec.a.ar_blockcount = len; error = xfs_btree_lookup(cur, XFS_LOOKUP_GE, stat); - cur->bc_private.a.priv.abt.active = (*stat == 1); + cur->bc_ag.abt.active = (*stat == 1); return error; } @@ -190,7 +190,7 @@ xfs_alloc_lookup_le( cur->bc_rec.a.ar_startblock = bno; cur->bc_rec.a.ar_blockcount = len; error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat); - cur->bc_private.a.priv.abt.active = (*stat == 1); + cur->bc_ag.abt.active = (*stat == 1); return error; } @@ -198,7 +198,7 @@ static inline bool xfs_alloc_cur_active( struct xfs_btree_cur *cur) { - return cur && cur->bc_private.a.priv.abt.active; + return cur && cur->bc_ag.abt.active; } /* @@ -230,7 +230,7 @@ xfs_alloc_get_rec( int *stat) /* output: success/failure */ { struct xfs_mount *mp = cur->bc_mp; - xfs_agnumber_t agno = cur->bc_private.a.agno; + xfs_agnumber_t agno = cur->bc_ag.agno; union xfs_btree_rec *rec; int error; @@ -589,6 +589,7 @@ xfs_agfl_verify( { struct xfs_mount *mp = bp->b_mount; struct xfs_agfl *agfl = XFS_BUF_TO_AGFL(bp); + __be32 *agfl_bno = xfs_buf_to_agfl_bno(bp); int i; /* @@ -614,8 +615,8 @@ xfs_agfl_verify( return __this_address; for (i = 0; i < xfs_agfl_size(mp); i++) { - if (be32_to_cpu(agfl->agfl_bno[i]) != NULLAGBLOCK && - be32_to_cpu(agfl->agfl_bno[i]) >= mp->m_sb.sb_agblocks) + if (be32_to_cpu(agfl_bno[i]) != NULLAGBLOCK && + be32_to_cpu(agfl_bno[i]) >= mp->m_sb.sb_agblocks) return __this_address; } @@ -713,7 +714,7 @@ xfs_alloc_update_counters( struct xfs_buf *agbp, long len) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_agf *agf = agbp->b_addr; pag->pagf_freeblks += len; be32_add_cpu(&agf->agf_freeblks, len); @@ -721,7 +722,7 @@ xfs_alloc_update_counters( xfs_trans_agblocks_delta(tp, len); if (unlikely(be32_to_cpu(agf->agf_freeblks) > be32_to_cpu(agf->agf_length))) { - xfs_buf_corruption_error(agbp); + xfs_buf_mark_corrupt(agbp); return -EFSCORRUPTED; } @@ -907,7 +908,7 @@ xfs_alloc_cur_check( deactivate = true; out: if (deactivate) - cur->bc_private.a.priv.abt.active = false; + cur->bc_ag.abt.active = false; trace_xfs_alloc_cur_check(args->mp, cur->bc_btnum, bno, len, diff, *new); return 0; @@ -922,13 +923,13 @@ xfs_alloc_cur_finish( struct xfs_alloc_arg *args, struct xfs_alloc_cur *acur) { + struct xfs_agf __maybe_unused *agf = args->agbp->b_addr; int error; ASSERT(acur->cnt && acur->bnolt); ASSERT(acur->bno >= acur->rec_bno); ASSERT(acur->bno + acur->len <= acur->rec_bno + acur->rec_len); - ASSERT(acur->rec_bno + acur->rec_len <= - be32_to_cpu(XFS_BUF_TO_AGF(args->agbp)->agf_length)); + ASSERT(acur->rec_bno + acur->rec_len <= be32_to_cpu(agf->agf_length)); error = xfs_alloc_fixup_trees(acur->cnt, acur->bnolt, acur->rec_bno, acur->rec_len, acur->bno, acur->len, 0); @@ -1026,6 +1027,7 @@ xfs_alloc_ag_vextent_small( xfs_extlen_t *flenp, /* result length */ int *stat) /* status: 0-freelist, 1-normal/none */ { + struct xfs_agf *agf = args->agbp->b_addr; int error = 0; xfs_agblock_t fbno = NULLAGBLOCK; xfs_extlen_t flen = 0; @@ -1054,8 +1056,7 @@ xfs_alloc_ag_vextent_small( if (args->minlen != 1 || args->alignment != 1 || args->resv == XFS_AG_RESV_AGFL || - (be32_to_cpu(XFS_BUF_TO_AGF(args->agbp)->agf_flcount) <= - args->minleft)) + be32_to_cpu(agf->agf_flcount) <= args->minleft) goto out; error = xfs_alloc_get_freelist(args->tp, args->agbp, &fbno, 0); @@ -1079,9 +1080,7 @@ xfs_alloc_ag_vextent_small( } *fbnop = args->agbno = fbno; *flenp = args->len = 1; - if (XFS_IS_CORRUPT(args->mp, - fbno >= be32_to_cpu( - XFS_BUF_TO_AGF(args->agbp)->agf_length))) { + if (XFS_IS_CORRUPT(args->mp, fbno >= be32_to_cpu(agf->agf_length))) { error = -EFSCORRUPTED; goto error; } @@ -1203,6 +1202,7 @@ STATIC int /* error */ xfs_alloc_ag_vextent_exact( xfs_alloc_arg_t *args) /* allocation argument structure */ { + struct xfs_agf __maybe_unused *agf = args->agbp->b_addr; xfs_btree_cur_t *bno_cur;/* by block-number btree cursor */ xfs_btree_cur_t *cnt_cur;/* by count btree cursor */ int error; @@ -1281,8 +1281,7 @@ xfs_alloc_ag_vextent_exact( */ cnt_cur = xfs_allocbt_init_cursor(args->mp, args->tp, args->agbp, args->agno, XFS_BTNUM_CNT); - ASSERT(args->agbno + args->len <= - be32_to_cpu(XFS_BUF_TO_AGF(args->agbp)->agf_length)); + ASSERT(args->agbno + args->len <= be32_to_cpu(agf->agf_length)); error = xfs_alloc_fixup_trees(cnt_cur, bno_cur, fbno, flen, args->agbno, args->len, XFSA_FIXUP_BNO_OK); if (error) { @@ -1353,7 +1352,7 @@ xfs_alloc_walk_iter( if (error) return error; if (i == 0) - cur->bc_private.a.priv.abt.active = false; + cur->bc_ag.abt.active = false; if (count > 0) count--; @@ -1468,7 +1467,7 @@ xfs_alloc_ag_vextent_locality( if (error) return error; if (i) { - acur->cnt->bc_private.a.priv.abt.active = true; + acur->cnt->bc_ag.abt.active = true; fbcur = acur->cnt; fbinc = false; } @@ -1515,7 +1514,7 @@ xfs_alloc_ag_vextent_lastblock( * maxlen, go to the start of this block, and skip all those smaller * than minlen. */ - if (len || args->alignment > 1) { + if (*len || args->alignment > 1) { acur->cnt->bc_ptrs[0] = 1; do { error = xfs_alloc_get_rec(acur->cnt, bno, len, &i); @@ -1661,6 +1660,7 @@ STATIC int /* error */ xfs_alloc_ag_vextent_size( xfs_alloc_arg_t *args) /* allocation argument structure */ { + struct xfs_agf *agf = args->agbp->b_addr; xfs_btree_cur_t *bno_cur; /* cursor for bno btree */ xfs_btree_cur_t *cnt_cur; /* cursor for cnt btree */ int error; /* error result */ @@ -1851,8 +1851,7 @@ xfs_alloc_ag_vextent_size( args->agbno = rbno; if (XFS_IS_CORRUPT(args->mp, args->agbno + args->len > - be32_to_cpu( - XFS_BUF_TO_AGF(args->agbp)->agf_length))) { + be32_to_cpu(agf->agf_length))) { error = -EFSCORRUPTED; goto error0; } @@ -2424,7 +2423,7 @@ xfs_agfl_reset( struct xfs_perag *pag) { struct xfs_mount *mp = tp->t_mountp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_agf *agf = agbp->b_addr; ASSERT(pag->pagf_agflreset); trace_xfs_agfl_reset(mp, agf, 0, _RET_IP_); @@ -2655,7 +2654,7 @@ xfs_alloc_get_freelist( xfs_agblock_t *bnop, /* block address retrieved from freelist */ int btreeblk) /* destination is a AGF btree */ { - xfs_agf_t *agf; /* a.g. freespace structure */ + struct xfs_agf *agf = agbp->b_addr; xfs_buf_t *agflbp;/* buffer for a.g. freelist structure */ xfs_agblock_t bno; /* block number returned */ __be32 *agfl_bno; @@ -2667,7 +2666,6 @@ xfs_alloc_get_freelist( /* * Freelist is empty, give up. */ - agf = XFS_BUF_TO_AGF(agbp); if (!agf->agf_flcount) { *bnop = NULLAGBLOCK; return 0; @@ -2684,7 +2682,7 @@ xfs_alloc_get_freelist( /* * Get the block number and update the data structures. */ - agfl_bno = XFS_BUF_TO_AGFL_BNO(mp, agflbp); + agfl_bno = xfs_buf_to_agfl_bno(agflbp); bno = be32_to_cpu(agfl_bno[be32_to_cpu(agf->agf_flfirst)]); be32_add_cpu(&agf->agf_flfirst, 1); xfs_trans_brelse(tp, agflbp); @@ -2745,7 +2743,7 @@ xfs_alloc_log_agf( sizeof(xfs_agf_t) }; - trace_xfs_agf(tp->t_mountp, XFS_BUF_TO_AGF(bp), fields, _RET_IP_); + trace_xfs_agf(tp->t_mountp, bp->b_addr, fields, _RET_IP_); xfs_trans_buf_set_type(tp, bp, XFS_BLFT_AGF_BUF); @@ -2783,18 +2781,15 @@ xfs_alloc_put_freelist( xfs_agblock_t bno, /* block being freed */ int btreeblk) /* block came from a AGF btree */ { - xfs_agf_t *agf; /* a.g. freespace structure */ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_agf *agf = agbp->b_addr; __be32 *blockp;/* pointer to array entry */ int error; int logflags; - xfs_mount_t *mp; /* mount structure */ xfs_perag_t *pag; /* per allocation group data */ __be32 *agfl_bno; int startoff; - agf = XFS_BUF_TO_AGF(agbp); - mp = tp->t_mountp; - if (!agflbp && (error = xfs_alloc_read_agfl(mp, tp, be32_to_cpu(agf->agf_seqno), &agflbp))) return error; @@ -2820,7 +2815,7 @@ xfs_alloc_put_freelist( ASSERT(be32_to_cpu(agf->agf_flcount) <= xfs_agfl_size(mp)); - agfl_bno = XFS_BUF_TO_AGFL_BNO(mp, agflbp); + agfl_bno = xfs_buf_to_agfl_bno(agflbp); blockp = &agfl_bno[be32_to_cpu(agf->agf_fllast)]; *blockp = cpu_to_be32(bno); startoff = (char *)blockp - (char *)agflbp->b_addr; @@ -2838,13 +2833,12 @@ xfs_agf_verify( struct xfs_buf *bp) { struct xfs_mount *mp = bp->b_mount; - struct xfs_agf *agf = XFS_BUF_TO_AGF(bp); + struct xfs_agf *agf = bp->b_addr; if (xfs_sb_version_hascrc(&mp->m_sb)) { if (!uuid_equal(&agf->agf_uuid, &mp->m_sb.sb_meta_uuid)) return __this_address; - if (!xfs_log_check_lsn(mp, - be64_to_cpu(XFS_BUF_TO_AGF(bp)->agf_lsn))) + if (!xfs_log_check_lsn(mp, be64_to_cpu(agf->agf_lsn))) return __this_address; } @@ -2858,6 +2852,13 @@ xfs_agf_verify( be32_to_cpu(agf->agf_flcount) <= xfs_agfl_size(mp))) return __this_address; + if (be32_to_cpu(agf->agf_length) > mp->m_sb.sb_dblocks) + return __this_address; + + if (be32_to_cpu(agf->agf_freeblks) < be32_to_cpu(agf->agf_longest) || + be32_to_cpu(agf->agf_freeblks) > be32_to_cpu(agf->agf_length)) + return __this_address; + if (be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]) < 1 || be32_to_cpu(agf->agf_levels[XFS_BTNUM_CNT]) < 1 || be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]) > XFS_BTREE_MAXLEVELS || @@ -2869,6 +2870,10 @@ xfs_agf_verify( be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]) > XFS_BTREE_MAXLEVELS)) return __this_address; + if (xfs_sb_version_hasrmapbt(&mp->m_sb) && + be32_to_cpu(agf->agf_rmap_blocks) > be32_to_cpu(agf->agf_length)) + return __this_address; + /* * during growfs operations, the perag is not fully initialised, * so we can't use it for any useful checking. growfs ensures we can't @@ -2882,6 +2887,11 @@ xfs_agf_verify( be32_to_cpu(agf->agf_btreeblks) > be32_to_cpu(agf->agf_length)) return __this_address; + if (xfs_sb_version_hasreflink(&mp->m_sb) && + be32_to_cpu(agf->agf_refcount_blocks) > + be32_to_cpu(agf->agf_length)) + return __this_address; + if (xfs_sb_version_hasreflink(&mp->m_sb) && (be32_to_cpu(agf->agf_refcount_level) < 1 || be32_to_cpu(agf->agf_refcount_level) > XFS_BTREE_MAXLEVELS)) @@ -2914,6 +2924,7 @@ xfs_agf_write_verify( { struct xfs_mount *mp = bp->b_mount; struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_agf *agf = bp->b_addr; xfs_failaddr_t fa; fa = xfs_agf_verify(bp); @@ -2926,7 +2937,7 @@ xfs_agf_write_verify( return; if (bip) - XFS_BUF_TO_AGF(bp)->agf_lsn = cpu_to_be64(bip->bli_item.li_lsn); + agf->agf_lsn = cpu_to_be64(bip->bli_item.li_lsn); xfs_buf_update_cksum(bp, XFS_AGF_CRC_OFF); } @@ -2994,7 +3005,7 @@ xfs_alloc_read_agf( return error; ASSERT(!(*bpp)->b_error); - agf = XFS_BUF_TO_AGF(*bpp); + agf = (*bpp)->b_addr; pag = xfs_perag_get(mp, agno); if (!pag->pagf_init) { pag->pagf_freeblks = be32_to_cpu(agf->agf_freeblks); @@ -3275,6 +3286,7 @@ __xfs_free_extent( struct xfs_buf *agbp; xfs_agnumber_t agno = XFS_FSB_TO_AGNO(mp, bno); xfs_agblock_t agbno = XFS_FSB_TO_AGBNO(mp, bno); + struct xfs_agf *agf; int error; unsigned int busy_flags = 0; @@ -3288,6 +3300,7 @@ __xfs_free_extent( error = xfs_free_extent_fix_freelist(tp, agno, &agbp); if (error) return error; + agf = agbp->b_addr; if (XFS_IS_CORRUPT(mp, agbno >= mp->m_sb.sb_agblocks)) { error = -EFSCORRUPTED; @@ -3295,9 +3308,7 @@ __xfs_free_extent( } /* validate the extent size is legal now we have the agf locked */ - if (XFS_IS_CORRUPT(mp, - agbno + len > - be32_to_cpu(XFS_BUF_TO_AGF(agbp)->agf_length))) { + if (XFS_IS_CORRUPT(mp, agbno + len > be32_to_cpu(agf->agf_length))) { error = -EFSCORRUPTED; goto err; } @@ -3408,7 +3419,7 @@ xfs_agfl_walk( unsigned int i; int error; - agfl_bno = XFS_BUF_TO_AGFL_BNO(mp, agflbp); + agfl_bno = xfs_buf_to_agfl_bno(agflbp); i = be32_to_cpu(agf->agf_flfirst); /* Nothing to walk in an empty AGFL. */ diff --git a/fs/xfs/libxfs/xfs_alloc.h b/fs/xfs/libxfs/xfs_alloc.h index 7380fbe4a3ff..a851bf77f17b 100644 --- a/fs/xfs/libxfs/xfs_alloc.h +++ b/fs/xfs/libxfs/xfs_alloc.h @@ -236,4 +236,13 @@ typedef int (*xfs_agfl_walk_fn)(struct xfs_mount *mp, xfs_agblock_t bno, int xfs_agfl_walk(struct xfs_mount *mp, struct xfs_agf *agf, struct xfs_buf *agflbp, xfs_agfl_walk_fn walk_fn, void *priv); +static inline __be32 * +xfs_buf_to_agfl_bno( + struct xfs_buf *bp) +{ + if (xfs_sb_version_hascrc(&bp->b_mount->m_sb)) + return bp->b_addr + sizeof(struct xfs_agfl); + return bp->b_addr; +} + #endif /* __XFS_ALLOC_H__ */ diff --git a/fs/xfs/libxfs/xfs_alloc_btree.c b/fs/xfs/libxfs/xfs_alloc_btree.c index 279694d73e4e..60c453cb3ee3 100644 --- a/fs/xfs/libxfs/xfs_alloc_btree.c +++ b/fs/xfs/libxfs/xfs_alloc_btree.c @@ -12,6 +12,7 @@ #include "xfs_sb.h" #include "xfs_mount.h" #include "xfs_btree.h" +#include "xfs_btree_staging.h" #include "xfs_alloc_btree.h" #include "xfs_alloc.h" #include "xfs_extent_busy.h" @@ -25,7 +26,7 @@ xfs_allocbt_dup_cursor( struct xfs_btree_cur *cur) { return xfs_allocbt_init_cursor(cur->bc_mp, cur->bc_tp, - cur->bc_private.a.agbp, cur->bc_private.a.agno, + cur->bc_ag.agbp, cur->bc_ag.agno, cur->bc_btnum); } @@ -35,8 +36,8 @@ xfs_allocbt_set_root( union xfs_btree_ptr *ptr, int inc) { - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; xfs_agnumber_t seqno = be32_to_cpu(agf->agf_seqno); int btnum = cur->bc_btnum; struct xfs_perag *pag = xfs_perag_get(cur->bc_mp, seqno); @@ -62,7 +63,7 @@ xfs_allocbt_alloc_block( xfs_agblock_t bno; /* Allocate the new block from the freelist. If we can't, give up. */ - error = xfs_alloc_get_freelist(cur->bc_tp, cur->bc_private.a.agbp, + error = xfs_alloc_get_freelist(cur->bc_tp, cur->bc_ag.agbp, &bno, 1); if (error) return error; @@ -72,7 +73,7 @@ xfs_allocbt_alloc_block( return 0; } - xfs_extent_busy_reuse(cur->bc_mp, cur->bc_private.a.agno, bno, 1, false); + xfs_extent_busy_reuse(cur->bc_mp, cur->bc_ag.agno, bno, 1, false); xfs_trans_agbtree_delta(cur->bc_tp, 1); new->s = cpu_to_be32(bno); @@ -86,8 +87,8 @@ xfs_allocbt_free_block( struct xfs_btree_cur *cur, struct xfs_buf *bp) { - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; xfs_agblock_t bno; int error; @@ -113,7 +114,7 @@ xfs_allocbt_update_lastrec( int ptr, int reason) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(cur->bc_private.a.agbp); + struct xfs_agf *agf = cur->bc_ag.agbp->b_addr; xfs_agnumber_t seqno = be32_to_cpu(agf->agf_seqno); struct xfs_perag *pag; __be32 len; @@ -162,7 +163,7 @@ xfs_allocbt_update_lastrec( pag = xfs_perag_get(cur->bc_mp, seqno); pag->pagf_longest = be32_to_cpu(len); xfs_perag_put(pag); - xfs_alloc_log_agf(cur->bc_tp, cur->bc_private.a.agbp, XFS_AGF_LONGEST); + xfs_alloc_log_agf(cur->bc_tp, cur->bc_ag.agbp, XFS_AGF_LONGEST); } STATIC int @@ -226,9 +227,9 @@ xfs_allocbt_init_ptr_from_cur( struct xfs_btree_cur *cur, union xfs_btree_ptr *ptr) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(cur->bc_private.a.agbp); + struct xfs_agf *agf = cur->bc_ag.agbp->b_addr; - ASSERT(cur->bc_private.a.agno == be32_to_cpu(agf->agf_seqno)); + ASSERT(cur->bc_ag.agno == be32_to_cpu(agf->agf_seqno)); ptr->s = agf->agf_roots[cur->bc_btnum]; } @@ -471,18 +472,14 @@ static const struct xfs_btree_ops xfs_cntbt_ops = { .recs_inorder = xfs_cntbt_recs_inorder, }; -/* - * Allocate a new allocation btree cursor. - */ -struct xfs_btree_cur * /* new alloc btree cursor */ -xfs_allocbt_init_cursor( - struct xfs_mount *mp, /* file system mount point */ - struct xfs_trans *tp, /* transaction pointer */ - struct xfs_buf *agbp, /* buffer for agf structure */ - xfs_agnumber_t agno, /* allocation group number */ - xfs_btnum_t btnum) /* btree identifier */ +/* Allocate most of a new allocation btree cursor. */ +STATIC struct xfs_btree_cur * +xfs_allocbt_init_common( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_agnumber_t agno, + xfs_btnum_t btnum) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); struct xfs_btree_cur *cur; ASSERT(btnum == XFS_BTNUM_BNO || btnum == XFS_BTNUM_CNT); @@ -495,19 +492,16 @@ xfs_allocbt_init_cursor( cur->bc_blocklog = mp->m_sb.sb_blocklog; if (btnum == XFS_BTNUM_CNT) { - cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_abtc_2); cur->bc_ops = &xfs_cntbt_ops; - cur->bc_nlevels = be32_to_cpu(agf->agf_levels[XFS_BTNUM_CNT]); + cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_abtc_2); cur->bc_flags = XFS_BTREE_LASTREC_UPDATE; } else { - cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_abtb_2); cur->bc_ops = &xfs_bnobt_ops; - cur->bc_nlevels = be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]); + cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_abtb_2); } - cur->bc_private.a.agbp = agbp; - cur->bc_private.a.agno = agno; - cur->bc_private.a.priv.abt.active = false; + cur->bc_ag.agno = agno; + cur->bc_ag.abt.active = false; if (xfs_sb_version_hascrc(&mp->m_sb)) cur->bc_flags |= XFS_BTREE_CRC_BLOCKS; @@ -515,6 +509,73 @@ xfs_allocbt_init_cursor( return cur; } +/* + * Allocate a new allocation btree cursor. + */ +struct xfs_btree_cur * /* new alloc btree cursor */ +xfs_allocbt_init_cursor( + struct xfs_mount *mp, /* file system mount point */ + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_buf *agbp, /* buffer for agf structure */ + xfs_agnumber_t agno, /* allocation group number */ + xfs_btnum_t btnum) /* btree identifier */ +{ + struct xfs_agf *agf = agbp->b_addr; + struct xfs_btree_cur *cur; + + cur = xfs_allocbt_init_common(mp, tp, agno, btnum); + if (btnum == XFS_BTNUM_CNT) + cur->bc_nlevels = be32_to_cpu(agf->agf_levels[XFS_BTNUM_CNT]); + else + cur->bc_nlevels = be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]); + + cur->bc_ag.agbp = agbp; + + return cur; +} + +/* Create a free space btree cursor with a fake root for staging. */ +struct xfs_btree_cur * +xfs_allocbt_stage_cursor( + struct xfs_mount *mp, + struct xbtree_afakeroot *afake, + xfs_agnumber_t agno, + xfs_btnum_t btnum) +{ + struct xfs_btree_cur *cur; + + cur = xfs_allocbt_init_common(mp, NULL, agno, btnum); + xfs_btree_stage_afakeroot(cur, afake); + return cur; +} + +/* + * Install a new free space btree root. Caller is responsible for invalidating + * and freeing the old btree blocks. + */ +void +xfs_allocbt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_agf *agf = agbp->b_addr; + struct xbtree_afakeroot *afake = cur->bc_ag.afake; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + agf->agf_roots[cur->bc_btnum] = cpu_to_be32(afake->af_root); + agf->agf_levels[cur->bc_btnum] = cpu_to_be32(afake->af_levels); + xfs_alloc_log_agf(tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS); + + if (cur->bc_btnum == XFS_BTNUM_BNO) { + xfs_btree_commit_afakeroot(cur, tp, agbp, &xfs_bnobt_ops); + } else { + cur->bc_flags |= XFS_BTREE_LASTREC_UPDATE; + xfs_btree_commit_afakeroot(cur, tp, agbp, &xfs_cntbt_ops); + } +} + /* * Calculate number of records in an alloc btree block. */ diff --git a/fs/xfs/libxfs/xfs_alloc_btree.h b/fs/xfs/libxfs/xfs_alloc_btree.h index c9305ebb69f6..047f09f0be3c 100644 --- a/fs/xfs/libxfs/xfs_alloc_btree.h +++ b/fs/xfs/libxfs/xfs_alloc_btree.h @@ -13,6 +13,7 @@ struct xfs_buf; struct xfs_btree_cur; struct xfs_mount; +struct xbtree_afakeroot; /* * Btree block header size depends on a superblock flag. @@ -48,8 +49,14 @@ struct xfs_mount; extern struct xfs_btree_cur *xfs_allocbt_init_cursor(struct xfs_mount *, struct xfs_trans *, struct xfs_buf *, xfs_agnumber_t, xfs_btnum_t); +struct xfs_btree_cur *xfs_allocbt_stage_cursor(struct xfs_mount *mp, + struct xbtree_afakeroot *afake, xfs_agnumber_t agno, + xfs_btnum_t btnum); extern int xfs_allocbt_maxrecs(struct xfs_mount *, int, int); extern xfs_extlen_t xfs_allocbt_calc_size(struct xfs_mount *mp, unsigned long long len); +void xfs_allocbt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp, struct xfs_buf *agbp); + #endif /* __XFS_ALLOC_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index e6149720ce02..e4fe3dca9883 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -56,33 +56,6 @@ STATIC int xfs_attr_node_removename(xfs_da_args_t *args); STATIC int xfs_attr_fillstate(xfs_da_state_t *state); STATIC int xfs_attr_refillstate(xfs_da_state_t *state); - -STATIC int -xfs_attr_args_init( - struct xfs_da_args *args, - struct xfs_inode *dp, - const unsigned char *name, - size_t namelen, - int flags) -{ - - if (!name) - return -EINVAL; - - memset(args, 0, sizeof(*args)); - args->geo = dp->i_mount->m_attr_geo; - args->whichfork = XFS_ATTR_FORK; - args->dp = dp; - args->flags = flags; - args->name = name; - args->namelen = namelen; - if (args->namelen >= MAXNAMELEN) - return -EFAULT; /* match IRIX behaviour */ - - args->hashval = xfs_da_hashname(args->name, args->namelen); - return 0; -} - int xfs_inode_hasattr( struct xfs_inode *ip) @@ -104,85 +77,60 @@ xfs_inode_hasattr( */ int xfs_attr_get_ilocked( - struct xfs_inode *ip, struct xfs_da_args *args) { - ASSERT(xfs_isilocked(ip, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); + ASSERT(xfs_isilocked(args->dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); - if (!xfs_inode_hasattr(ip)) + if (!xfs_inode_hasattr(args->dp)) return -ENOATTR; - else if (ip->i_d.di_aformat == XFS_DINODE_FMT_LOCAL) + + if (args->dp->i_d.di_aformat == XFS_DINODE_FMT_LOCAL) return xfs_attr_shortform_getvalue(args); - else if (xfs_bmap_one_block(ip, XFS_ATTR_FORK)) + if (xfs_bmap_one_block(args->dp, XFS_ATTR_FORK)) return xfs_attr_leaf_get(args); - else - return xfs_attr_node_get(args); + return xfs_attr_node_get(args); } /* * Retrieve an extended attribute by name, and its value if requested. * - * If ATTR_KERNOVAL is set in @flags, then the caller does not want the value, - * just an indication whether the attribute exists and the size of the value if - * it exists. The size is returned in @valuelenp, + * If args->valuelen is zero, then the caller does not want the value, just an + * indication whether the attribute exists and the size of the value if it + * exists. The size is returned in args.valuelen. + * + * If args->value is NULL but args->valuelen is non-zero, allocate the buffer + * for the value after existence of the attribute has been determined. The + * caller always has to free args->value if it is set, no matter if this + * function was successful or not. * * If the attribute is found, but exceeds the size limit set by the caller in - * @valuelenp, return -ERANGE with the size of the attribute that was found in - * @valuelenp. - * - * If ATTR_ALLOC is set in @flags, allocate the buffer for the value after - * existence of the attribute has been determined. On success, return that - * buffer to the caller and leave them to free it. On failure, free any - * allocated buffer and ensure the buffer pointer returned to the caller is - * null. + * args->valuelen, return -ERANGE with the size of the attribute that was found + * in args->valuelen. */ int xfs_attr_get( - struct xfs_inode *ip, - const unsigned char *name, - size_t namelen, - unsigned char **value, - int *valuelenp, - int flags) + struct xfs_da_args *args) { - struct xfs_da_args args; uint lock_mode; int error; - ASSERT((flags & (ATTR_ALLOC | ATTR_KERNOVAL)) || *value); + XFS_STATS_INC(args->dp->i_mount, xs_attr_get); - XFS_STATS_INC(ip->i_mount, xs_attr_get); - - if (XFS_FORCED_SHUTDOWN(ip->i_mount)) + if (XFS_FORCED_SHUTDOWN(args->dp->i_mount)) return -EIO; - error = xfs_attr_args_init(&args, ip, name, namelen, flags); - if (error) - return error; + args->geo = args->dp->i_mount->m_attr_geo; + args->whichfork = XFS_ATTR_FORK; + args->hashval = xfs_da_hashname(args->name, args->namelen); /* Entirely possible to look up a name which doesn't exist */ - args.op_flags = XFS_DA_OP_OKNOENT; - if (flags & ATTR_ALLOC) - args.op_flags |= XFS_DA_OP_ALLOCVAL; - else - args.value = *value; - args.valuelen = *valuelenp; + args->op_flags = XFS_DA_OP_OKNOENT; - lock_mode = xfs_ilock_attr_map_shared(ip); - error = xfs_attr_get_ilocked(ip, &args); - xfs_iunlock(ip, lock_mode); - *valuelenp = args.valuelen; + lock_mode = xfs_ilock_attr_map_shared(args->dp); + error = xfs_attr_get_ilocked(args); + xfs_iunlock(args->dp, lock_mode); - /* on error, we have to clean up allocated value buffers */ - if (error) { - if (flags & ATTR_ALLOC) { - kmem_free(args.value); - *value = NULL; - } - return error; - } - *value = args.value; - return 0; + return error; } /* @@ -238,7 +186,7 @@ xfs_attr_try_sf_addname( * Commit the shortform mods, and we're done. * NOTE: this is also the error path (EEXIST, etc). */ - if (!error && (args->flags & ATTR_KERNOTIME) == 0) + if (!error && !(args->op_flags & XFS_DA_OP_NOTIME)) xfs_trans_ichgtime(args->trans, dp, XFS_ICHGTIME_CHG); if (mp->m_flags & XFS_MOUNT_WSYNC) @@ -336,80 +284,102 @@ xfs_attr_remove_args( return error; } +/* + * Note: If args->value is NULL the attribute will be removed, just like the + * Linux ->setattr API. + */ int xfs_attr_set( - struct xfs_inode *dp, - const unsigned char *name, - size_t namelen, - unsigned char *value, - int valuelen, - int flags) + struct xfs_da_args *args) { + struct xfs_inode *dp = args->dp; struct xfs_mount *mp = dp->i_mount; - struct xfs_da_args args; struct xfs_trans_res tres; - int rsvd = (flags & ATTR_ROOT) != 0; + bool rsvd = (args->attr_filter & XFS_ATTR_ROOT); int error, local; - - XFS_STATS_INC(mp, xs_attr_set); + unsigned int total; if (XFS_FORCED_SHUTDOWN(dp->i_mount)) return -EIO; - error = xfs_attr_args_init(&args, dp, name, namelen, flags); - if (error) - return error; - - args.value = value; - args.valuelen = valuelen; - args.op_flags = XFS_DA_OP_ADDNAME | XFS_DA_OP_OKNOENT; - args.total = xfs_attr_calc_size(&args, &local); - error = xfs_qm_dqattach(dp); if (error) return error; + args->geo = mp->m_attr_geo; + args->whichfork = XFS_ATTR_FORK; + args->hashval = xfs_da_hashname(args->name, args->namelen); + /* - * If the inode doesn't have an attribute fork, add one. - * (inode must not be locked when we call this routine) + * We have no control over the attribute names that userspace passes us + * to remove, so we have to allow the name lookup prior to attribute + * removal to fail as well. */ - if (XFS_IFORK_Q(dp) == 0) { - int sf_size = sizeof(xfs_attr_sf_hdr_t) + - XFS_ATTR_SF_ENTSIZE_BYNAME(args.namelen, valuelen); + args->op_flags = XFS_DA_OP_OKNOENT; - error = xfs_bmap_add_attrfork(dp, sf_size, rsvd); - if (error) - return error; + if (args->value) { + XFS_STATS_INC(mp, xs_attr_set); + + args->op_flags |= XFS_DA_OP_ADDNAME; + args->total = xfs_attr_calc_size(args, &local); + + /* + * If the inode doesn't have an attribute fork, add one. + * (inode must not be locked when we call this routine) + */ + if (XFS_IFORK_Q(dp) == 0) { + int sf_size = sizeof(struct xfs_attr_sf_hdr) + + XFS_ATTR_SF_ENTSIZE_BYNAME(args->namelen, + args->valuelen); + + error = xfs_bmap_add_attrfork(dp, sf_size, rsvd); + if (error) + return error; + } + + tres.tr_logres = M_RES(mp)->tr_attrsetm.tr_logres + + M_RES(mp)->tr_attrsetrt.tr_logres * + args->total; + tres.tr_logcount = XFS_ATTRSET_LOG_COUNT; + tres.tr_logflags = XFS_TRANS_PERM_LOG_RES; + total = args->total; + } else { + XFS_STATS_INC(mp, xs_attr_remove); + + tres = M_RES(mp)->tr_attrrm; + total = XFS_ATTRRM_SPACE_RES(mp); } - tres.tr_logres = M_RES(mp)->tr_attrsetm.tr_logres + - M_RES(mp)->tr_attrsetrt.tr_logres * args.total; - tres.tr_logcount = XFS_ATTRSET_LOG_COUNT; - tres.tr_logflags = XFS_TRANS_PERM_LOG_RES; - /* * Root fork attributes can use reserved data blocks for this * operation if necessary */ - error = xfs_trans_alloc(mp, &tres, args.total, 0, - rsvd ? XFS_TRANS_RESERVE : 0, &args.trans); + error = xfs_trans_alloc(mp, &tres, total, 0, + rsvd ? XFS_TRANS_RESERVE : 0, &args->trans); if (error) return error; xfs_ilock(dp, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota_nblks(args.trans, dp, args.total, 0, - rsvd ? XFS_QMOPT_RES_REGBLKS | XFS_QMOPT_FORCE_RES : - XFS_QMOPT_RES_REGBLKS); - if (error) - goto out_trans_cancel; + xfs_trans_ijoin(args->trans, dp, 0); + if (args->value) { + unsigned int quota_flags = XFS_QMOPT_RES_REGBLKS; - xfs_trans_ijoin(args.trans, dp, 0); - error = xfs_attr_set_args(&args); - if (error) - goto out_trans_cancel; - if (!args.trans) { + if (rsvd) + quota_flags |= XFS_QMOPT_FORCE_RES; + error = xfs_trans_reserve_quota_nblks(args->trans, dp, + args->total, 0, quota_flags); + if (error) + goto out_trans_cancel; + error = xfs_attr_set_args(args); + if (error) + goto out_trans_cancel; /* shortform attribute has already been committed */ - goto out_unlock; + if (!args->trans) + goto out_unlock; + } else { + error = xfs_attr_remove_args(args); + if (error) + goto out_trans_cancel; } /* @@ -417,109 +387,26 @@ xfs_attr_set( * transaction goes to disk before returning to the user. */ if (mp->m_flags & XFS_MOUNT_WSYNC) - xfs_trans_set_sync(args.trans); + xfs_trans_set_sync(args->trans); - if ((flags & ATTR_KERNOTIME) == 0) - xfs_trans_ichgtime(args.trans, dp, XFS_ICHGTIME_CHG); + if (!(args->op_flags & XFS_DA_OP_NOTIME)) + xfs_trans_ichgtime(args->trans, dp, XFS_ICHGTIME_CHG); /* * Commit the last in the sequence of transactions. */ - xfs_trans_log_inode(args.trans, dp, XFS_ILOG_CORE); - error = xfs_trans_commit(args.trans); + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE); + error = xfs_trans_commit(args->trans); out_unlock: xfs_iunlock(dp, XFS_ILOCK_EXCL); return error; out_trans_cancel: - if (args.trans) - xfs_trans_cancel(args.trans); + if (args->trans) + xfs_trans_cancel(args->trans); goto out_unlock; } -/* - * Generic handler routine to remove a name from an attribute list. - * Transitions attribute list from Btree to shortform as necessary. - */ -int -xfs_attr_remove( - struct xfs_inode *dp, - const unsigned char *name, - size_t namelen, - int flags) -{ - struct xfs_mount *mp = dp->i_mount; - struct xfs_da_args args; - int error; - - XFS_STATS_INC(mp, xs_attr_remove); - - if (XFS_FORCED_SHUTDOWN(dp->i_mount)) - return -EIO; - - error = xfs_attr_args_init(&args, dp, name, namelen, flags); - if (error) - return error; - - /* - * we have no control over the attribute names that userspace passes us - * to remove, so we have to allow the name lookup prior to attribute - * removal to fail. - */ - args.op_flags = XFS_DA_OP_OKNOENT; - - error = xfs_qm_dqattach(dp); - if (error) - return error; - - /* - * Root fork attributes can use reserved data blocks for this - * operation if necessary - */ - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_attrrm, - XFS_ATTRRM_SPACE_RES(mp), 0, - (flags & ATTR_ROOT) ? XFS_TRANS_RESERVE : 0, - &args.trans); - if (error) - return error; - - xfs_ilock(dp, XFS_ILOCK_EXCL); - /* - * No need to make quota reservations here. We expect to release some - * blocks not allocate in the common case. - */ - xfs_trans_ijoin(args.trans, dp, 0); - - error = xfs_attr_remove_args(&args); - if (error) - goto out; - - /* - * If this is a synchronous mount, make sure that the - * transaction goes to disk before returning to the user. - */ - if (mp->m_flags & XFS_MOUNT_WSYNC) - xfs_trans_set_sync(args.trans); - - if ((flags & ATTR_KERNOTIME) == 0) - xfs_trans_ichgtime(args.trans, dp, XFS_ICHGTIME_CHG); - - /* - * Commit the last in the sequence of transactions. - */ - xfs_trans_log_inode(args.trans, dp, XFS_ILOG_CORE); - error = xfs_trans_commit(args.trans); - xfs_iunlock(dp, XFS_ILOCK_EXCL); - - return error; - -out: - if (args.trans) - xfs_trans_cancel(args.trans); - xfs_iunlock(dp, XFS_ILOCK_EXCL); - return error; -} - /*======================================================================== * External routines when attribute list is inside the inode *========================================================================*/ @@ -536,10 +423,10 @@ xfs_attr_shortform_addname(xfs_da_args_t *args) trace_xfs_attr_sf_addname(args); retval = xfs_attr_shortform_lookup(args); - if ((args->flags & ATTR_REPLACE) && (retval == -ENOATTR)) { + if (retval == -ENOATTR && (args->attr_flags & XATTR_REPLACE)) return retval; - } else if (retval == -EEXIST) { - if (args->flags & ATTR_CREATE) + if (retval == -EEXIST) { + if (args->attr_flags & XATTR_CREATE) return retval; retval = xfs_attr_shortform_remove(args); if (retval) @@ -549,7 +436,7 @@ xfs_attr_shortform_addname(xfs_da_args_t *args) * that the leaf format add routine won't trip over the attr * not being around. */ - args->flags &= ~ATTR_REPLACE; + args->attr_flags &= ~XATTR_REPLACE; } if (args->namelen >= XFS_ATTR_SF_ENTSIZE_MAX || @@ -602,14 +489,11 @@ xfs_attr_leaf_addname( * the given flags produce an error or call for an atomic rename. */ retval = xfs_attr3_leaf_lookup_int(bp, args); - if ((args->flags & ATTR_REPLACE) && (retval == -ENOATTR)) { - xfs_trans_brelse(args->trans, bp); - return retval; - } else if (retval == -EEXIST) { - if (args->flags & ATTR_CREATE) { /* pure create op */ - xfs_trans_brelse(args->trans, bp); - return retval; - } + if (retval == -ENOATTR && (args->attr_flags & XATTR_REPLACE)) + goto out_brelse; + if (retval == -EEXIST) { + if (args->attr_flags & XATTR_CREATE) + goto out_brelse; trace_xfs_attr_leaf_replace(args); @@ -750,6 +634,9 @@ xfs_attr_leaf_addname( error = xfs_attr3_leaf_clearflag(args); } return error; +out_brelse: + xfs_trans_brelse(args->trans, bp); + return retval; } /* @@ -876,10 +763,10 @@ xfs_attr_node_addname( goto out; blk = &state->path.blk[ state->path.active-1 ]; ASSERT(blk->magic == XFS_ATTR_LEAF_MAGIC); - if ((args->flags & ATTR_REPLACE) && (retval == -ENOATTR)) { + if (retval == -ENOATTR && (args->attr_flags & XATTR_REPLACE)) goto out; - } else if (retval == -EEXIST) { - if (args->flags & ATTR_CREATE) + if (retval == -EEXIST) { + if (args->attr_flags & XATTR_CREATE) goto out; trace_xfs_attr_node_replace(args); @@ -1011,7 +898,7 @@ xfs_attr_node_addname( * The INCOMPLETE flag means that we will find the "old" * attr, not the "new" one. */ - args->op_flags |= XFS_DA_OP_INCOMPLETE; + args->attr_filter |= XFS_ATTR_INCOMPLETE; state = xfs_da_state_alloc(); state->args = args; state->mp = mp; diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index 4243b2272642..0d2d05908537 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -21,39 +21,6 @@ struct xfs_attr_list_context; * as possible so as to fit into the literal area of the inode. */ -/*======================================================================== - * External interfaces - *========================================================================*/ - - -#define ATTR_DONTFOLLOW 0x0001 /* -- ignored, from IRIX -- */ -#define ATTR_ROOT 0x0002 /* use attrs in root (trusted) namespace */ -#define ATTR_TRUST 0x0004 /* -- unused, from IRIX -- */ -#define ATTR_SECURE 0x0008 /* use attrs in security namespace */ -#define ATTR_CREATE 0x0010 /* pure create: fail if attr already exists */ -#define ATTR_REPLACE 0x0020 /* pure set: fail if attr does not exist */ - -#define ATTR_KERNOTIME 0x1000 /* [kernel] don't update inode timestamps */ -#define ATTR_KERNOVAL 0x2000 /* [kernel] get attr size only, not value */ - -#define ATTR_INCOMPLETE 0x4000 /* [kernel] return INCOMPLETE attr keys */ -#define ATTR_ALLOC 0x8000 /* [kernel] allocate xattr buffer on demand */ - -#define ATTR_KERNEL_FLAGS \ - (ATTR_KERNOTIME | ATTR_KERNOVAL | ATTR_INCOMPLETE | ATTR_ALLOC) - -#define XFS_ATTR_FLAGS \ - { ATTR_DONTFOLLOW, "DONTFOLLOW" }, \ - { ATTR_ROOT, "ROOT" }, \ - { ATTR_TRUST, "TRUST" }, \ - { ATTR_SECURE, "SECURE" }, \ - { ATTR_CREATE, "CREATE" }, \ - { ATTR_REPLACE, "REPLACE" }, \ - { ATTR_KERNOTIME, "KERNOTIME" }, \ - { ATTR_KERNOVAL, "KERNOVAL" }, \ - { ATTR_INCOMPLETE, "INCOMPLETE" }, \ - { ATTR_ALLOC, "ALLOC" } - /* * The maximum size (into the kernel or returned from the kernel) of an * attribute value or the buffer used for an attr_list() call. Larger @@ -61,46 +28,17 @@ struct xfs_attr_list_context; */ #define ATTR_MAX_VALUELEN (64*1024) /* max length of a value */ -/* - * Define how lists of attribute names are returned to the user from - * the attr_list() call. A large, 32bit aligned, buffer is passed in - * along with its size. We put an array of offsets at the top that each - * reference an attrlist_ent_t and pack the attrlist_ent_t's at the bottom. - */ -typedef struct attrlist { - __s32 al_count; /* number of entries in attrlist */ - __s32 al_more; /* T/F: more attrs (do call again) */ - __s32 al_offset[1]; /* byte offsets of attrs [var-sized] */ -} attrlist_t; - -/* - * Show the interesting info about one attribute. This is what the - * al_offset[i] entry points to. - */ -typedef struct attrlist_ent { /* data from attr_list() */ - __u32 a_valuelen; /* number bytes in value of attr */ - char a_name[1]; /* attr name (NULL terminated) */ -} attrlist_ent_t; - -/* - * Given a pointer to the (char*) buffer containing the attr_list() result, - * and an index, return a pointer to the indicated attribute in the buffer. - */ -#define ATTR_ENTRY(buffer, index) \ - ((attrlist_ent_t *) \ - &((char *)buffer)[ ((attrlist_t *)(buffer))->al_offset[index] ]) - /* * Kernel-internal version of the attrlist cursor. */ -typedef struct attrlist_cursor_kern { +struct xfs_attrlist_cursor_kern { __u32 hashval; /* hash value of next entry to add */ __u32 blkno; /* block containing entry (suggestion) */ __u32 offset; /* offset in list of equal-hashvals */ __u16 pad1; /* padding to match user-level */ __u8 pad2; /* padding to match user-level */ __u8 initted; /* T/F: cursor has been initialized */ -} attrlist_cursor_kern_t; +}; /*======================================================================== @@ -112,27 +50,28 @@ typedef struct attrlist_cursor_kern { typedef void (*put_listent_func_t)(struct xfs_attr_list_context *, int, unsigned char *, int, int); -typedef struct xfs_attr_list_context { - struct xfs_trans *tp; - struct xfs_inode *dp; /* inode */ - struct attrlist_cursor_kern *cursor; /* position in list */ - char *alist; /* output buffer */ +struct xfs_attr_list_context { + struct xfs_trans *tp; + struct xfs_inode *dp; /* inode */ + struct xfs_attrlist_cursor_kern cursor; /* position in list */ + void *buffer; /* output buffer */ /* * Abort attribute list iteration if non-zero. Can be used to pass * error values to the xfs_attr_list caller. */ - int seen_enough; + int seen_enough; + bool allow_incomplete; - ssize_t count; /* num used entries */ - int dupcnt; /* count dup hashvals seen */ - int bufsize; /* total buffer size */ - int firstu; /* first used byte in buffer */ - int flags; /* from VOP call */ - int resynch; /* T/F: resynch with cursor */ - put_listent_func_t put_listent; /* list output fmt function */ - int index; /* index into output buffer */ -} xfs_attr_list_context_t; + ssize_t count; /* num used entries */ + int dupcnt; /* count dup hashvals seen */ + int bufsize; /* total buffer size */ + int firstu; /* first used byte in buffer */ + unsigned int attr_filter; /* XFS_ATTR_{ROOT,SECURE} */ + int resynch; /* T/F: resynch with cursor */ + put_listent_func_t put_listent; /* list output fmt function */ + int index; /* index into output buffer */ +}; /*======================================================================== @@ -143,21 +82,14 @@ typedef struct xfs_attr_list_context { * Overall external interface routines. */ int xfs_attr_inactive(struct xfs_inode *dp); -int xfs_attr_list_int_ilocked(struct xfs_attr_list_context *); -int xfs_attr_list_int(struct xfs_attr_list_context *); +int xfs_attr_list_ilocked(struct xfs_attr_list_context *); +int xfs_attr_list(struct xfs_attr_list_context *); int xfs_inode_hasattr(struct xfs_inode *ip); -int xfs_attr_get_ilocked(struct xfs_inode *ip, struct xfs_da_args *args); -int xfs_attr_get(struct xfs_inode *ip, const unsigned char *name, - size_t namelen, unsigned char **value, int *valuelenp, - int flags); -int xfs_attr_set(struct xfs_inode *dp, const unsigned char *name, - size_t namelen, unsigned char *value, int valuelen, int flags); +int xfs_attr_get_ilocked(struct xfs_da_args *args); +int xfs_attr_get(struct xfs_da_args *args); +int xfs_attr_set(struct xfs_da_args *args); int xfs_attr_set_args(struct xfs_da_args *args); -int xfs_attr_remove(struct xfs_inode *dp, const unsigned char *name, - size_t namelen, int flags); int xfs_attr_remove_args(struct xfs_da_args *args); -int xfs_attr_list(struct xfs_inode *dp, char *buffer, int bufsize, - int flags, struct attrlist_cursor_kern *cursor); bool xfs_attr_namecheck(const void *name, size_t length); #endif /* __XFS_ATTR_H__ */ diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index fed537a4353d..863444e2dda7 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -445,14 +445,25 @@ xfs_attr3_leaf_read( * Namespace helper routines *========================================================================*/ -/* - * If namespace bits don't match return 0. - * If all match then return 1. - */ -STATIC int -xfs_attr_namesp_match(int arg_flags, int ondisk_flags) +static bool +xfs_attr_match( + struct xfs_da_args *args, + uint8_t namelen, + unsigned char *name, + int flags) { - return XFS_ATTR_NSP_ONDISK(ondisk_flags) == XFS_ATTR_NSP_ARGS_TO_ONDISK(arg_flags); + if (args->namelen != namelen) + return false; + if (memcmp(args->name, name, namelen) != 0) + return false; + /* + * If we are looking for incomplete entries, show only those, else only + * show complete entries. + */ + if (args->attr_filter != + (flags & (XFS_ATTR_NSP_ONDISK_MASK | XFS_ATTR_INCOMPLETE))) + return false; + return true; } static int @@ -464,7 +475,7 @@ xfs_attr_copy_value( /* * No copy if all we have to do is get the length */ - if (args->flags & ATTR_KERNOVAL) { + if (!args->valuelen) { args->valuelen = valuelen; return 0; } @@ -477,7 +488,7 @@ xfs_attr_copy_value( return -ERANGE; } - if (args->op_flags & XFS_DA_OP_ALLOCVAL) { + if (!args->value) { args->value = kmem_alloc_large(valuelen, 0); if (!args->value) return -ENOMEM; @@ -526,7 +537,7 @@ xfs_attr_shortform_bytesfit( int offset; /* rounded down */ - offset = (XFS_LITINO(mp, dp->i_d.di_version) - bytes) >> 3; + offset = (XFS_LITINO(mp) - bytes) >> 3; if (dp->i_d.di_format == XFS_DINODE_FMT_DEV) { minforkoff = roundup(sizeof(xfs_dev_t), 8) >> 3; @@ -593,8 +604,7 @@ xfs_attr_shortform_bytesfit( minforkoff = roundup(minforkoff, 8) >> 3; /* attr fork btree root can have at least this many key/ptr pairs */ - maxforkoff = XFS_LITINO(mp, dp->i_d.di_version) - - XFS_BMDR_SPACE_CALC(MINABTPTRS); + maxforkoff = XFS_LITINO(mp) - XFS_BMDR_SPACE_CALC(MINABTPTRS); maxforkoff = maxforkoff >> 3; /* rounded down */ if (offset >= maxforkoff) @@ -678,15 +688,8 @@ xfs_attr_shortform_add(xfs_da_args_t *args, int forkoff) sf = (xfs_attr_shortform_t *)ifp->if_u1.if_data; sfe = &sf->list[0]; for (i = 0; i < sf->hdr.count; sfe = XFS_ATTR_SF_NEXTENTRY(sfe), i++) { -#ifdef DEBUG - if (sfe->namelen != args->namelen) - continue; - if (memcmp(args->name, sfe->nameval, args->namelen) != 0) - continue; - if (!xfs_attr_namesp_match(args->flags, sfe->flags)) - continue; - ASSERT(0); -#endif + ASSERT(!xfs_attr_match(args, sfe->namelen, sfe->nameval, + sfe->flags)); } offset = (char *)sfe - (char *)sf; @@ -697,7 +700,7 @@ xfs_attr_shortform_add(xfs_da_args_t *args, int forkoff) sfe->namelen = args->namelen; sfe->valuelen = args->valuelen; - sfe->flags = XFS_ATTR_NSP_ARGS_TO_ONDISK(args->flags); + sfe->flags = args->attr_filter; memcpy(sfe->nameval, args->name, args->namelen); memcpy(&sfe->nameval[args->namelen], args->value, args->valuelen); sf->hdr.count++; @@ -749,13 +752,9 @@ xfs_attr_shortform_remove(xfs_da_args_t *args) for (i = 0; i < end; sfe = XFS_ATTR_SF_NEXTENTRY(sfe), base += size, i++) { size = XFS_ATTR_SF_ENTSIZE(sfe); - if (sfe->namelen != args->namelen) - continue; - if (memcmp(sfe->nameval, args->name, args->namelen) != 0) - continue; - if (!xfs_attr_namesp_match(args->flags, sfe->flags)) - continue; - break; + if (xfs_attr_match(args, sfe->namelen, sfe->nameval, + sfe->flags)) + break; } if (i == end) return -ENOATTR; @@ -816,13 +815,9 @@ xfs_attr_shortform_lookup(xfs_da_args_t *args) sfe = &sf->list[0]; for (i = 0; i < sf->hdr.count; sfe = XFS_ATTR_SF_NEXTENTRY(sfe), i++) { - if (sfe->namelen != args->namelen) - continue; - if (memcmp(args->name, sfe->nameval, args->namelen) != 0) - continue; - if (!xfs_attr_namesp_match(args->flags, sfe->flags)) - continue; - return -EEXIST; + if (xfs_attr_match(args, sfe->namelen, sfe->nameval, + sfe->flags)) + return -EEXIST; } return -ENOATTR; } @@ -830,9 +825,9 @@ xfs_attr_shortform_lookup(xfs_da_args_t *args) /* * Retrieve the attribute value and length. * - * If ATTR_KERNOVAL is specified, only the length needs to be returned. - * Unlike a lookup, we only return an error if the attribute does not - * exist or we can't retrieve the value. + * If args->valuelen is zero, only the length needs to be returned. Unlike a + * lookup, we only return an error if the attribute does not exist or we can't + * retrieve the value. */ int xfs_attr_shortform_getvalue( @@ -847,14 +842,10 @@ xfs_attr_shortform_getvalue( sfe = &sf->list[0]; for (i = 0; i < sf->hdr.count; sfe = XFS_ATTR_SF_NEXTENTRY(sfe), i++) { - if (sfe->namelen != args->namelen) - continue; - if (memcmp(args->name, sfe->nameval, args->namelen) != 0) - continue; - if (!xfs_attr_namesp_match(args->flags, sfe->flags)) - continue; - return xfs_attr_copy_value(args, &sfe->nameval[args->namelen], - sfe->valuelen); + if (xfs_attr_match(args, sfe->namelen, sfe->nameval, + sfe->flags)) + return xfs_attr_copy_value(args, + &sfe->nameval[args->namelen], sfe->valuelen); } return -ENOATTR; } @@ -918,7 +909,7 @@ xfs_attr_shortform_to_leaf( nargs.valuelen = sfe->valuelen; nargs.hashval = xfs_da_hashname(sfe->nameval, sfe->namelen); - nargs.flags = XFS_ATTR_NSP_ONDISK_TO_ARGS(sfe->flags); + nargs.attr_filter = sfe->flags & XFS_ATTR_NSP_ONDISK_MASK; error = xfs_attr3_leaf_lookup_int(bp, &nargs); /* set a->index */ ASSERT(error == -ENOATTR); error = xfs_attr3_leaf_add(bp, &nargs); @@ -1124,7 +1115,7 @@ xfs_attr3_leaf_to_shortform( nargs.value = &name_loc->nameval[nargs.namelen]; nargs.valuelen = be16_to_cpu(name_loc->valuelen); nargs.hashval = be32_to_cpu(entry->hashval); - nargs.flags = XFS_ATTR_NSP_ONDISK_TO_ARGS(entry->flags); + nargs.attr_filter = entry->flags & XFS_ATTR_NSP_ONDISK_MASK; xfs_attr_shortform_add(&nargs, forkoff); } error = 0; @@ -1449,8 +1440,9 @@ xfs_attr3_leaf_add_work( entry->nameidx = cpu_to_be16(ichdr->freemap[mapindex].base + ichdr->freemap[mapindex].size); entry->hashval = cpu_to_be32(args->hashval); - entry->flags = tmp ? XFS_ATTR_LOCAL : 0; - entry->flags |= XFS_ATTR_NSP_ARGS_TO_ONDISK(args->flags); + entry->flags = args->attr_filter; + if (tmp) + entry->flags |= XFS_ATTR_LOCAL; if (args->op_flags & XFS_DA_OP_RENAME) { entry->flags |= XFS_ATTR_INCOMPLETE; if ((args->blkno2 == args->blkno) && @@ -2346,7 +2338,7 @@ xfs_attr3_leaf_lookup_int( xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr, leaf); entries = xfs_attr3_leaf_entryp(leaf); if (ichdr.count >= args->geo->blksize / 8) { - xfs_buf_corruption_error(bp); + xfs_buf_mark_corrupt(bp); return -EFSCORRUPTED; } @@ -2365,11 +2357,11 @@ xfs_attr3_leaf_lookup_int( break; } if (!(probe >= 0 && (!ichdr.count || probe < ichdr.count))) { - xfs_buf_corruption_error(bp); + xfs_buf_mark_corrupt(bp); return -EFSCORRUPTED; } if (!(span <= 4 || be32_to_cpu(entry->hashval) == hashval)) { - xfs_buf_corruption_error(bp); + xfs_buf_mark_corrupt(bp); return -EFSCORRUPTED; } @@ -2399,33 +2391,17 @@ xfs_attr3_leaf_lookup_int( /* * GROT: Add code to remove incomplete entries. */ - /* - * If we are looking for INCOMPLETE entries, show only those. - * If we are looking for complete entries, show only those. - */ - if (!!(args->op_flags & XFS_DA_OP_INCOMPLETE) != - !!(entry->flags & XFS_ATTR_INCOMPLETE)) { - continue; - } if (entry->flags & XFS_ATTR_LOCAL) { name_loc = xfs_attr3_leaf_name_local(leaf, probe); - if (name_loc->namelen != args->namelen) - continue; - if (memcmp(args->name, name_loc->nameval, - args->namelen) != 0) - continue; - if (!xfs_attr_namesp_match(args->flags, entry->flags)) + if (!xfs_attr_match(args, name_loc->namelen, + name_loc->nameval, entry->flags)) continue; args->index = probe; return -EEXIST; } else { name_rmt = xfs_attr3_leaf_name_remote(leaf, probe); - if (name_rmt->namelen != args->namelen) - continue; - if (memcmp(args->name, name_rmt->name, - args->namelen) != 0) - continue; - if (!xfs_attr_namesp_match(args->flags, entry->flags)) + if (!xfs_attr_match(args, name_rmt->namelen, + name_rmt->name, entry->flags)) continue; args->index = probe; args->rmtvaluelen = be32_to_cpu(name_rmt->valuelen); @@ -2444,9 +2420,9 @@ xfs_attr3_leaf_lookup_int( * Get the value associated with an attribute name from a leaf attribute * list structure. * - * If ATTR_KERNOVAL is specified, only the length needs to be returned. - * Unlike a lookup, we only return an error if the attribute does not - * exist or we can't retrieve the value. + * If args->valuelen is zero, only the length needs to be returned. Unlike a + * lookup, we only return an error if the attribute does not exist or we can't + * retrieve the value. */ int xfs_attr3_leaf_getvalue( diff --git a/fs/xfs/libxfs/xfs_attr_leaf.h b/fs/xfs/libxfs/xfs_attr_leaf.h index 73615b1dd1a8..6dd2d937a42a 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.h +++ b/fs/xfs/libxfs/xfs_attr_leaf.h @@ -8,7 +8,6 @@ #define __XFS_ATTR_LEAF_H__ struct attrlist; -struct attrlist_cursor_kern; struct xfs_attr_list_context; struct xfs_da_args; struct xfs_da_state; diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c index 8b7f74b3bea2..01ad7f353e08 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.c +++ b/fs/xfs/libxfs/xfs_attr_remote.c @@ -397,7 +397,7 @@ xfs_attr_rmtval_get( trace_xfs_attr_rmtval_get(args); - ASSERT(!(args->flags & ATTR_KERNOVAL)); + ASSERT(args->valuelen != 0); ASSERT(args->rmtvaluelen == args->valuelen); valuelen = args->rmtvaluelen; diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 9a6d7a84689a..fda13cd7add0 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -193,14 +193,12 @@ xfs_default_attroffset( struct xfs_mount *mp = ip->i_mount; uint offset; - if (mp->m_sb.sb_inodesize == 256) { - offset = XFS_LITINO(mp, ip->i_d.di_version) - - XFS_BMDR_SPACE_CALC(MINABTPTRS); - } else { + if (mp->m_sb.sb_inodesize == 256) + offset = XFS_LITINO(mp) - XFS_BMDR_SPACE_CALC(MINABTPTRS); + else offset = XFS_BMDR_SPACE_CALC(6 * MINABTPTRS); - } - ASSERT(offset < XFS_LITINO(mp, ip->i_d.di_version)); + ASSERT(offset < XFS_LITINO(mp)); return offset; } @@ -690,7 +688,7 @@ xfs_bmap_extents_to_btree( * Need a cursor. Can't allocate until bb_level is filled in. */ cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); - cur->bc_private.b.flags = wasdel ? XFS_BTCUR_BPRV_WASDEL : 0; + cur->bc_ino.flags = wasdel ? XFS_BTCUR_BMBT_WASDEL : 0; /* * Convert to a btree with two levels, one record in root. */ @@ -727,7 +725,7 @@ xfs_bmap_extents_to_btree( ASSERT(tp->t_firstblock == NULLFSBLOCK || args.agno >= XFS_FSB_TO_AGNO(mp, tp->t_firstblock)); tp->t_firstblock = args.fsbno; - cur->bc_private.b.allocated++; + cur->bc_ino.allocated++; ip->i_d.di_nblocks++; xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1L); error = xfs_trans_get_buf(tp, mp->m_ddev_targp, @@ -953,7 +951,7 @@ xfs_bmap_add_attrfork_btree( xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); return -ENOSPC; } - cur->bc_private.b.allocated = 0; + cur->bc_ino.allocated = 0; xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); } return 0; @@ -980,7 +978,7 @@ xfs_bmap_add_attrfork_extents( error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0, flags, XFS_DATA_FORK); if (cur) { - cur->bc_private.b.allocated = 0; + cur->bc_ino.allocated = 0; xfs_btree_del_cursor(cur, error); } return error; @@ -1178,13 +1176,13 @@ xfs_iread_bmbt_block( { struct xfs_iread_state *ir = priv; struct xfs_mount *mp = cur->bc_mp; - struct xfs_inode *ip = cur->bc_private.b.ip; + struct xfs_inode *ip = cur->bc_ino.ip; struct xfs_btree_block *block; struct xfs_buf *bp; struct xfs_bmbt_rec *frp; xfs_extnum_t num_recs; xfs_extnum_t j; - int whichfork = cur->bc_private.b.whichfork; + int whichfork = cur->bc_ino.whichfork; block = xfs_btree_get_block(cur, level, &bp); @@ -1528,7 +1526,7 @@ xfs_bmap_add_extent_delay_real( ASSERT(!isnullstartblock(new->br_startblock)); ASSERT(!bma->cur || - (bma->cur->bc_private.b.flags & XFS_BTCUR_BPRV_WASDEL)); + (bma->cur->bc_ino.flags & XFS_BTCUR_BMBT_WASDEL)); XFS_STATS_INC(mp, xs_add_exlist); @@ -1818,7 +1816,7 @@ xfs_bmap_add_extent_delay_real( temp = PREV.br_blockcount - new->br_blockcount; da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp), startblockval(PREV.br_startblock) - - (bma->cur ? bma->cur->bc_private.b.allocated : 0)); + (bma->cur ? bma->cur->bc_ino.allocated : 0)); PREV.br_startoff = new_endoff; PREV.br_blockcount = temp; @@ -1904,7 +1902,7 @@ xfs_bmap_add_extent_delay_real( temp = PREV.br_blockcount - new->br_blockcount; da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp), startblockval(PREV.br_startblock) - - (bma->cur ? bma->cur->bc_private.b.allocated : 0)); + (bma->cur ? bma->cur->bc_ino.allocated : 0)); PREV.br_startblock = nullstartblock(da_new); PREV.br_blockcount = temp; @@ -2025,8 +2023,8 @@ xfs_bmap_add_extent_delay_real( xfs_mod_delalloc(mp, (int64_t)da_new - da_old); if (bma->cur) { - da_new += bma->cur->bc_private.b.allocated; - bma->cur->bc_private.b.allocated = 0; + da_new += bma->cur->bc_ino.allocated; + bma->cur->bc_ino.allocated = 0; } /* adjust for changes in reserved delayed indirect blocks */ @@ -2573,7 +2571,7 @@ xfs_bmap_add_extent_unwritten_real( /* clear out the allocated field, done with it now in any case. */ if (cur) { - cur->bc_private.b.allocated = 0; + cur->bc_ino.allocated = 0; *curp = cur; } @@ -2752,7 +2750,7 @@ xfs_bmap_add_extent_hole_real( struct xfs_bmbt_irec old; ASSERT(!isnullstartblock(new->br_startblock)); - ASSERT(!cur || !(cur->bc_private.b.flags & XFS_BTCUR_BPRV_WASDEL)); + ASSERT(!cur || !(cur->bc_ino.flags & XFS_BTCUR_BMBT_WASDEL)); XFS_STATS_INC(mp, xs_add_exlist); @@ -2955,7 +2953,7 @@ xfs_bmap_add_extent_hole_real( /* clear out the allocated field, done with it now in any case. */ if (cur) - cur->bc_private.b.allocated = 0; + cur->bc_ino.allocated = 0; xfs_bmap_check_leaf_extents(cur, ip, whichfork); done: @@ -4187,8 +4185,8 @@ xfs_bmapi_allocate( bma->nallocs++; if (bma->cur) - bma->cur->bc_private.b.flags = - bma->wasdel ? XFS_BTCUR_BPRV_WASDEL : 0; + bma->cur->bc_ino.flags = + bma->wasdel ? XFS_BTCUR_BMBT_WASDEL : 0; bma->got.br_startoff = bma->offset; bma->got.br_startblock = bma->blkno; @@ -4709,7 +4707,7 @@ xfs_bmapi_remap( if (ifp->if_flags & XFS_IFBROOT) { cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); - cur->bc_private.b.flags = 0; + cur->bc_ino.flags = 0; } got.br_startoff = bno; @@ -5364,7 +5362,7 @@ __xfs_bunmapi( if (ifp->if_flags & XFS_IFBROOT) { ASSERT(XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_BTREE); cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); - cur->bc_private.b.flags = 0; + cur->bc_ino.flags = 0; } else cur = NULL; @@ -5620,7 +5618,7 @@ __xfs_bunmapi( xfs_trans_log_inode(tp, ip, logflags); if (cur) { if (!error) - cur->bc_private.b.allocated = 0; + cur->bc_ino.allocated = 0; xfs_btree_del_cursor(cur, error); } return error; @@ -5839,7 +5837,7 @@ xfs_bmap_collapse_extents( if (ifp->if_flags & XFS_IFBROOT) { cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); - cur->bc_private.b.flags = 0; + cur->bc_ino.flags = 0; } if (!xfs_iext_lookup_extent(ip, ifp, *next_fsb, &icur, &got)) { @@ -5956,7 +5954,7 @@ xfs_bmap_insert_extents( if (ifp->if_flags & XFS_IFBROOT) { cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); - cur->bc_private.b.flags = 0; + cur->bc_ino.flags = 0; } if (*next_fsb == NULLFSBLOCK) { @@ -6025,8 +6023,8 @@ xfs_bmap_insert_extents( * @split_fsb is a block where the extents is split. If split_fsb lies in a * hole or the first block of extents, just return 0. */ -STATIC int -xfs_bmap_split_extent_at( +int +xfs_bmap_split_extent( struct xfs_trans *tp, struct xfs_inode *ip, xfs_fileoff_t split_fsb) @@ -6074,7 +6072,7 @@ xfs_bmap_split_extent_at( if (ifp->if_flags & XFS_IFBROOT) { cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); - cur->bc_private.b.flags = 0; + cur->bc_ino.flags = 0; error = xfs_bmbt_lookup_eq(cur, &got, &i); if (error) goto del_cursor; @@ -6133,7 +6131,7 @@ xfs_bmap_split_extent_at( del_cursor: if (cur) { - cur->bc_private.b.allocated = 0; + cur->bc_ino.allocated = 0; xfs_btree_del_cursor(cur, error); } @@ -6142,34 +6140,6 @@ xfs_bmap_split_extent_at( return error; } -int -xfs_bmap_split_extent( - struct xfs_inode *ip, - xfs_fileoff_t split_fsb) -{ - struct xfs_mount *mp = ip->i_mount; - struct xfs_trans *tp; - int error; - - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, - XFS_DIOSTRAT_SPACE_RES(mp, 0), 0, 0, &tp); - if (error) - return error; - - xfs_ilock(ip, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); - - error = xfs_bmap_split_extent_at(tp, ip, split_fsb); - if (error) - goto out; - - return xfs_trans_commit(tp); - -out: - xfs_trans_cancel(tp); - return error; -} - /* Deferred mapping is only for real extents in the data fork. */ static bool xfs_bmap_is_update_needed( diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h index 14d25e0b7d9c..f3259ad5c22c 100644 --- a/fs/xfs/libxfs/xfs_bmap.h +++ b/fs/xfs/libxfs/xfs_bmap.h @@ -222,7 +222,8 @@ int xfs_bmap_can_insert_extents(struct xfs_inode *ip, xfs_fileoff_t off, int xfs_bmap_insert_extents(struct xfs_trans *tp, struct xfs_inode *ip, xfs_fileoff_t *next_fsb, xfs_fileoff_t offset_shift_fsb, bool *done, xfs_fileoff_t stop_fsb); -int xfs_bmap_split_extent(struct xfs_inode *ip, xfs_fileoff_t split_offset); +int xfs_bmap_split_extent(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_fileoff_t split_offset); int xfs_bmapi_reserve_delalloc(struct xfs_inode *ip, int whichfork, xfs_fileoff_t off, xfs_filblks_t len, xfs_filblks_t prealloc, struct xfs_bmbt_irec *got, struct xfs_iext_cursor *cur, diff --git a/fs/xfs/libxfs/xfs_bmap_btree.c b/fs/xfs/libxfs/xfs_bmap_btree.c index ffe608d2a2d9..295a59cf8840 100644 --- a/fs/xfs/libxfs/xfs_bmap_btree.c +++ b/fs/xfs/libxfs/xfs_bmap_btree.c @@ -166,13 +166,13 @@ xfs_bmbt_dup_cursor( struct xfs_btree_cur *new; new = xfs_bmbt_init_cursor(cur->bc_mp, cur->bc_tp, - cur->bc_private.b.ip, cur->bc_private.b.whichfork); + cur->bc_ino.ip, cur->bc_ino.whichfork); /* * Copy the firstblock, dfops, and flags values, * since init cursor doesn't get them. */ - new->bc_private.b.flags = cur->bc_private.b.flags; + new->bc_ino.flags = cur->bc_ino.flags; return new; } @@ -183,12 +183,12 @@ xfs_bmbt_update_cursor( struct xfs_btree_cur *dst) { ASSERT((dst->bc_tp->t_firstblock != NULLFSBLOCK) || - (dst->bc_private.b.ip->i_d.di_flags & XFS_DIFLAG_REALTIME)); + (dst->bc_ino.ip->i_d.di_flags & XFS_DIFLAG_REALTIME)); - dst->bc_private.b.allocated += src->bc_private.b.allocated; + dst->bc_ino.allocated += src->bc_ino.allocated; dst->bc_tp->t_firstblock = src->bc_tp->t_firstblock; - src->bc_private.b.allocated = 0; + src->bc_ino.allocated = 0; } STATIC int @@ -205,8 +205,8 @@ xfs_bmbt_alloc_block( args.tp = cur->bc_tp; args.mp = cur->bc_mp; args.fsbno = cur->bc_tp->t_firstblock; - xfs_rmap_ino_bmbt_owner(&args.oinfo, cur->bc_private.b.ip->i_ino, - cur->bc_private.b.whichfork); + xfs_rmap_ino_bmbt_owner(&args.oinfo, cur->bc_ino.ip->i_ino, + cur->bc_ino.whichfork); if (args.fsbno == NULLFSBLOCK) { args.fsbno = be64_to_cpu(start->l); @@ -230,7 +230,7 @@ xfs_bmbt_alloc_block( } args.minlen = args.maxlen = args.prod = 1; - args.wasdel = cur->bc_private.b.flags & XFS_BTCUR_BPRV_WASDEL; + args.wasdel = cur->bc_ino.flags & XFS_BTCUR_BMBT_WASDEL; if (!args.wasdel && args.tp->t_blk_res == 0) { error = -ENOSPC; goto error0; @@ -259,10 +259,10 @@ xfs_bmbt_alloc_block( ASSERT(args.len == 1); cur->bc_tp->t_firstblock = args.fsbno; - cur->bc_private.b.allocated++; - cur->bc_private.b.ip->i_d.di_nblocks++; - xfs_trans_log_inode(args.tp, cur->bc_private.b.ip, XFS_ILOG_CORE); - xfs_trans_mod_dquot_byino(args.tp, cur->bc_private.b.ip, + cur->bc_ino.allocated++; + cur->bc_ino.ip->i_d.di_nblocks++; + xfs_trans_log_inode(args.tp, cur->bc_ino.ip, XFS_ILOG_CORE); + xfs_trans_mod_dquot_byino(args.tp, cur->bc_ino.ip, XFS_TRANS_DQ_BCOUNT, 1L); new->l = cpu_to_be64(args.fsbno); @@ -280,12 +280,12 @@ xfs_bmbt_free_block( struct xfs_buf *bp) { struct xfs_mount *mp = cur->bc_mp; - struct xfs_inode *ip = cur->bc_private.b.ip; + struct xfs_inode *ip = cur->bc_ino.ip; struct xfs_trans *tp = cur->bc_tp; xfs_fsblock_t fsbno = XFS_DADDR_TO_FSB(mp, XFS_BUF_ADDR(bp)); struct xfs_owner_info oinfo; - xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, cur->bc_private.b.whichfork); + xfs_rmap_ino_bmbt_owner(&oinfo, ip->i_ino, cur->bc_ino.whichfork); xfs_bmap_add_free(cur->bc_tp, fsbno, 1, &oinfo); ip->i_d.di_nblocks--; @@ -302,8 +302,8 @@ xfs_bmbt_get_minrecs( if (level == cur->bc_nlevels - 1) { struct xfs_ifork *ifp; - ifp = XFS_IFORK_PTR(cur->bc_private.b.ip, - cur->bc_private.b.whichfork); + ifp = XFS_IFORK_PTR(cur->bc_ino.ip, + cur->bc_ino.whichfork); return xfs_bmbt_maxrecs(cur->bc_mp, ifp->if_broot_bytes, level == 0) / 2; @@ -320,8 +320,8 @@ xfs_bmbt_get_maxrecs( if (level == cur->bc_nlevels - 1) { struct xfs_ifork *ifp; - ifp = XFS_IFORK_PTR(cur->bc_private.b.ip, - cur->bc_private.b.whichfork); + ifp = XFS_IFORK_PTR(cur->bc_ino.ip, + cur->bc_ino.whichfork); return xfs_bmbt_maxrecs(cur->bc_mp, ifp->if_broot_bytes, level == 0); @@ -347,7 +347,7 @@ xfs_bmbt_get_dmaxrecs( { if (level != cur->bc_nlevels - 1) return cur->bc_mp->m_bmap_dmxr[level != 0]; - return xfs_bmdr_maxrecs(cur->bc_private.b.forksize, level == 0); + return xfs_bmdr_maxrecs(cur->bc_ino.forksize, level == 0); } STATIC void @@ -566,11 +566,11 @@ xfs_bmbt_init_cursor( if (xfs_sb_version_hascrc(&mp->m_sb)) cur->bc_flags |= XFS_BTREE_CRC_BLOCKS; - cur->bc_private.b.forksize = XFS_IFORK_SIZE(ip, whichfork); - cur->bc_private.b.ip = ip; - cur->bc_private.b.allocated = 0; - cur->bc_private.b.flags = 0; - cur->bc_private.b.whichfork = whichfork; + cur->bc_ino.forksize = XFS_IFORK_SIZE(ip, whichfork); + cur->bc_ino.ip = ip; + cur->bc_ino.allocated = 0; + cur->bc_ino.flags = 0; + cur->bc_ino.whichfork = whichfork; return cur; } @@ -644,7 +644,7 @@ xfs_bmbt_change_owner( cur = xfs_bmbt_init_cursor(ip->i_mount, tp, ip, whichfork); if (!cur) return -ENOMEM; - cur->bc_private.b.flags |= XFS_BTCUR_BPRV_INVALID_OWNER; + cur->bc_ino.flags |= XFS_BTCUR_BMBT_INVALID_OWNER; error = xfs_btree_change_owner(cur, new_owner, buffer_list); xfs_btree_del_cursor(cur, error); diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c index fd300dc93ca4..2d25bab68764 100644 --- a/fs/xfs/libxfs/xfs_btree.c +++ b/fs/xfs/libxfs/xfs_btree.c @@ -20,6 +20,7 @@ #include "xfs_trace.h" #include "xfs_alloc.h" #include "xfs_log.h" +#include "xfs_btree_staging.h" /* * Cursor allocation zone. @@ -214,7 +215,7 @@ xfs_btree_check_sptr( { if (level <= 0) return false; - return xfs_verify_agbno(cur->bc_mp, cur->bc_private.a.agno, agbno); + return xfs_verify_agbno(cur->bc_mp, cur->bc_ag.agno, agbno); } /* @@ -234,8 +235,8 @@ xfs_btree_check_ptr( return 0; xfs_err(cur->bc_mp, "Inode %llu fork %d: Corrupt btree %d pointer at level %d index %d.", - cur->bc_private.b.ip->i_ino, - cur->bc_private.b.whichfork, cur->bc_btnum, + cur->bc_ino.ip->i_ino, + cur->bc_ino.whichfork, cur->bc_btnum, level, index); } else { if (xfs_btree_check_sptr(cur, be32_to_cpu((&ptr->s)[index]), @@ -243,7 +244,7 @@ xfs_btree_check_ptr( return 0; xfs_err(cur->bc_mp, "AG %u: Corrupt btree %d pointer at level %d index %d.", - cur->bc_private.a.agno, cur->bc_btnum, + cur->bc_ag.agno, cur->bc_btnum, level, index); } @@ -378,10 +379,12 @@ xfs_btree_del_cursor( * allocated indirect blocks' accounting. */ ASSERT(cur->bc_btnum != XFS_BTNUM_BMAP || - cur->bc_private.b.allocated == 0); + cur->bc_ino.allocated == 0); /* * Free the cursor. */ + if (unlikely(cur->bc_flags & XFS_BTREE_STAGING)) + kmem_free((void *)cur->bc_ops); kmem_cache_free(xfs_btree_cur_zone, cur); } @@ -642,6 +645,17 @@ xfs_btree_ptr_addr( ((char *)block + xfs_btree_ptr_offset(cur, n, level)); } +struct xfs_ifork * +xfs_btree_ifork_ptr( + struct xfs_btree_cur *cur) +{ + ASSERT(cur->bc_flags & XFS_BTREE_ROOT_IN_INODE); + + if (cur->bc_flags & XFS_BTREE_STAGING) + return cur->bc_ino.ifake->if_fork; + return XFS_IFORK_PTR(cur->bc_ino.ip, cur->bc_ino.whichfork); +} + /* * Get the root block which is stored in the inode. * @@ -652,9 +666,8 @@ STATIC struct xfs_btree_block * xfs_btree_get_iroot( struct xfs_btree_cur *cur) { - struct xfs_ifork *ifp; + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); - ifp = XFS_IFORK_PTR(cur->bc_private.b.ip, cur->bc_private.b.whichfork); return (struct xfs_btree_block *)ifp->if_broot; } @@ -881,13 +894,13 @@ xfs_btree_readahead_sblock( if ((lr & XFS_BTCUR_LEFTRA) && left != NULLAGBLOCK) { - xfs_btree_reada_bufs(cur->bc_mp, cur->bc_private.a.agno, + xfs_btree_reada_bufs(cur->bc_mp, cur->bc_ag.agno, left, 1, cur->bc_ops->buf_ops); rval++; } if ((lr & XFS_BTCUR_RIGHTRA) && right != NULLAGBLOCK) { - xfs_btree_reada_bufs(cur->bc_mp, cur->bc_private.a.agno, + xfs_btree_reada_bufs(cur->bc_mp, cur->bc_ag.agno, right, 1, cur->bc_ops->buf_ops); rval++; } @@ -945,7 +958,7 @@ xfs_btree_ptr_to_daddr( *daddr = XFS_FSB_TO_DADDR(cur->bc_mp, fsbno); } else { agbno = be32_to_cpu(ptr->s); - *daddr = XFS_AGB_TO_DADDR(cur->bc_mp, cur->bc_private.a.agno, + *daddr = XFS_AGB_TO_DADDR(cur->bc_mp, cur->bc_ag.agno, agbno); } @@ -1014,7 +1027,7 @@ xfs_btree_ptr_is_null( return ptr->s == cpu_to_be32(NULLAGBLOCK); } -STATIC void +void xfs_btree_set_ptr_null( struct xfs_btree_cur *cur, union xfs_btree_ptr *ptr) @@ -1050,7 +1063,7 @@ xfs_btree_get_sibling( } } -STATIC void +void xfs_btree_set_sibling( struct xfs_btree_cur *cur, struct xfs_btree_block *block, @@ -1128,7 +1141,7 @@ xfs_btree_init_block( btnum, level, numrecs, owner, 0); } -STATIC void +void xfs_btree_init_block_cur( struct xfs_btree_cur *cur, struct xfs_buf *bp, @@ -1144,9 +1157,9 @@ xfs_btree_init_block_cur( * code. */ if (cur->bc_flags & XFS_BTREE_LONG_PTRS) - owner = cur->bc_private.b.ip->i_ino; + owner = cur->bc_ino.ip->i_ino; else - owner = cur->bc_private.a.agno; + owner = cur->bc_ag.agno; xfs_btree_init_block_int(cur->bc_mp, XFS_BUF_TO_BLOCK(bp), bp->b_bn, cur->bc_btnum, level, numrecs, @@ -1220,7 +1233,7 @@ xfs_btree_set_refs( } } -STATIC int +int xfs_btree_get_buf_block( struct xfs_btree_cur *cur, union xfs_btree_ptr *ptr, @@ -1280,7 +1293,7 @@ xfs_btree_read_buf_block( /* * Copy keys from one btree block to another. */ -STATIC void +void xfs_btree_copy_keys( struct xfs_btree_cur *cur, union xfs_btree_key *dst_key, @@ -1308,11 +1321,11 @@ xfs_btree_copy_recs( /* * Copy block pointers from one btree block to another. */ -STATIC void +void xfs_btree_copy_ptrs( struct xfs_btree_cur *cur, union xfs_btree_ptr *dst_ptr, - union xfs_btree_ptr *src_ptr, + const union xfs_btree_ptr *src_ptr, int numptrs) { ASSERT(numptrs >= 0); @@ -1393,8 +1406,8 @@ xfs_btree_log_keys( xfs_btree_key_offset(cur, first), xfs_btree_key_offset(cur, last + 1) - 1); } else { - xfs_trans_log_inode(cur->bc_tp, cur->bc_private.b.ip, - xfs_ilog_fbroot(cur->bc_private.b.whichfork)); + xfs_trans_log_inode(cur->bc_tp, cur->bc_ino.ip, + xfs_ilog_fbroot(cur->bc_ino.whichfork)); } } @@ -1436,8 +1449,8 @@ xfs_btree_log_ptrs( xfs_btree_ptr_offset(cur, first, level), xfs_btree_ptr_offset(cur, last + 1, level) - 1); } else { - xfs_trans_log_inode(cur->bc_tp, cur->bc_private.b.ip, - xfs_ilog_fbroot(cur->bc_private.b.whichfork)); + xfs_trans_log_inode(cur->bc_tp, cur->bc_ino.ip, + xfs_ilog_fbroot(cur->bc_ino.whichfork)); } } @@ -1505,8 +1518,8 @@ xfs_btree_log_block( xfs_trans_buf_set_type(cur->bc_tp, bp, XFS_BLFT_BTREE_BUF); xfs_trans_log_buf(cur->bc_tp, bp, first, last); } else { - xfs_trans_log_inode(cur->bc_tp, cur->bc_private.b.ip, - xfs_ilog_fbroot(cur->bc_private.b.whichfork)); + xfs_trans_log_inode(cur->bc_tp, cur->bc_ino.ip, + xfs_ilog_fbroot(cur->bc_ino.whichfork)); } } @@ -1743,10 +1756,10 @@ xfs_btree_lookup_get_block( /* Check the inode owner since the verifiers don't. */ if (xfs_sb_version_hascrc(&cur->bc_mp->m_sb) && - !(cur->bc_private.b.flags & XFS_BTCUR_BPRV_INVALID_OWNER) && + !(cur->bc_ino.flags & XFS_BTCUR_BMBT_INVALID_OWNER) && (cur->bc_flags & XFS_BTREE_LONG_PTRS) && be64_to_cpu((*blkp)->bb_u.l.bb_owner) != - cur->bc_private.b.ip->i_ino) + cur->bc_ino.ip->i_ino) goto out_bad; /* Did we get the level we were looking for? */ @@ -1762,7 +1775,7 @@ xfs_btree_lookup_get_block( out_bad: *blkp = NULL; - xfs_buf_corruption_error(bp); + xfs_buf_mark_corrupt(bp); xfs_trans_brelse(cur->bc_tp, bp); return -EFSCORRUPTED; } @@ -2938,9 +2951,9 @@ xfs_btree_new_iroot( xfs_btree_copy_ptrs(cur, pp, &nptr, 1); - xfs_iroot_realloc(cur->bc_private.b.ip, + xfs_iroot_realloc(cur->bc_ino.ip, 1 - xfs_btree_get_numrecs(cblock), - cur->bc_private.b.whichfork); + cur->bc_ino.whichfork); xfs_btree_setbuf(cur, level, cbp); @@ -2953,7 +2966,7 @@ xfs_btree_new_iroot( xfs_btree_log_ptrs(cur, cbp, 1, be16_to_cpu(cblock->bb_numrecs)); *logflags |= - XFS_ILOG_CORE | xfs_ilog_fbroot(cur->bc_private.b.whichfork); + XFS_ILOG_CORE | xfs_ilog_fbroot(cur->bc_ino.whichfork); *stat = 1; return 0; error0: @@ -3105,11 +3118,11 @@ xfs_btree_make_block_unfull( if ((cur->bc_flags & XFS_BTREE_ROOT_IN_INODE) && level == cur->bc_nlevels - 1) { - struct xfs_inode *ip = cur->bc_private.b.ip; + struct xfs_inode *ip = cur->bc_ino.ip; if (numrecs < cur->bc_ops->get_dmaxrecs(cur, level)) { /* A root block that can be made bigger. */ - xfs_iroot_realloc(ip, 1, cur->bc_private.b.whichfork); + xfs_iroot_realloc(ip, 1, cur->bc_ino.whichfork); *stat = 1; } else { /* A root block that needs replacing */ @@ -3455,8 +3468,8 @@ STATIC int xfs_btree_kill_iroot( struct xfs_btree_cur *cur) { - int whichfork = cur->bc_private.b.whichfork; - struct xfs_inode *ip = cur->bc_private.b.ip; + int whichfork = cur->bc_ino.whichfork; + struct xfs_inode *ip = cur->bc_ino.ip; struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); struct xfs_btree_block *block; struct xfs_btree_block *cblock; @@ -3514,8 +3527,8 @@ xfs_btree_kill_iroot( index = numrecs - cur->bc_ops->get_maxrecs(cur, level); if (index) { - xfs_iroot_realloc(cur->bc_private.b.ip, index, - cur->bc_private.b.whichfork); + xfs_iroot_realloc(cur->bc_ino.ip, index, + cur->bc_ino.whichfork); block = ifp->if_broot; } @@ -3544,7 +3557,7 @@ xfs_btree_kill_iroot( cur->bc_bufs[level - 1] = NULL; be16_add_cpu(&block->bb_level, -1); xfs_trans_log_inode(cur->bc_tp, ip, - XFS_ILOG_CORE | xfs_ilog_fbroot(cur->bc_private.b.whichfork)); + XFS_ILOG_CORE | xfs_ilog_fbroot(cur->bc_ino.whichfork)); cur->bc_nlevels--; out0: return 0; @@ -3712,8 +3725,8 @@ xfs_btree_delrec( */ if (level == cur->bc_nlevels - 1) { if (cur->bc_flags & XFS_BTREE_ROOT_IN_INODE) { - xfs_iroot_realloc(cur->bc_private.b.ip, -1, - cur->bc_private.b.whichfork); + xfs_iroot_realloc(cur->bc_ino.ip, -1, + cur->bc_ino.whichfork); error = xfs_btree_kill_iroot(cur); if (error) diff --git a/fs/xfs/libxfs/xfs_btree.h b/fs/xfs/libxfs/xfs_btree.h index 3eff7c321d43..8626c5a81aad 100644 --- a/fs/xfs/libxfs/xfs_btree.h +++ b/fs/xfs/libxfs/xfs_btree.h @@ -10,6 +10,7 @@ struct xfs_buf; struct xfs_inode; struct xfs_mount; struct xfs_trans; +struct xfs_ifork; extern kmem_zone_t *xfs_btree_cur_zone; @@ -177,15 +178,37 @@ union xfs_btree_irec { struct xfs_refcount_irec rc; }; -/* Per-AG btree private information. */ -union xfs_btree_cur_private { - struct { - unsigned long nr_ops; /* # record updates */ - int shape_changes; /* # of extent splits */ - } refc; - struct { - bool active; /* allocation cursor state */ - } abt; +/* Per-AG btree information. */ +struct xfs_btree_cur_ag { + union { + struct xfs_buf *agbp; + struct xbtree_afakeroot *afake; /* for staging cursor */ + }; + xfs_agnumber_t agno; + union { + struct { + unsigned long nr_ops; /* # record updates */ + int shape_changes; /* # of extent splits */ + } refc; + struct { + bool active; /* allocation cursor state */ + } abt; + }; +}; + +/* Btree-in-inode cursor information */ +struct xfs_btree_cur_ino { + struct xfs_inode *ip; + struct xbtree_ifakeroot *ifake; /* for staging cursor */ + int allocated; + short forksize; + char whichfork; + char flags; +/* We are converting a delalloc reservation */ +#define XFS_BTCUR_BMBT_WASDEL (1 << 0) + +/* For extent swap, ignore owner check in verifier */ +#define XFS_BTCUR_BMBT_INVALID_OWNER (1 << 1) }; /* @@ -209,21 +232,9 @@ typedef struct xfs_btree_cur xfs_btnum_t bc_btnum; /* identifies which btree type */ int bc_statoff; /* offset of btre stats array */ union { - struct { /* needed for BNO, CNT, INO */ - struct xfs_buf *agbp; /* agf/agi buffer pointer */ - xfs_agnumber_t agno; /* ag number */ - union xfs_btree_cur_private priv; - } a; - struct { /* needed for BMAP */ - struct xfs_inode *ip; /* pointer to our inode */ - int allocated; /* count of alloced */ - short forksize; /* fork's inode space */ - char whichfork; /* data or attr fork */ - char flags; /* flags */ -#define XFS_BTCUR_BPRV_WASDEL (1<<0) /* was delayed */ -#define XFS_BTCUR_BPRV_INVALID_OWNER (1<<1) /* for ext swap */ - } b; - } bc_private; /* per-btree type data */ + struct xfs_btree_cur_ag bc_ag; + struct xfs_btree_cur_ino bc_ino; + }; } xfs_btree_cur_t; /* cursor flags */ @@ -232,6 +243,12 @@ typedef struct xfs_btree_cur #define XFS_BTREE_LASTREC_UPDATE (1<<2) /* track last rec externally */ #define XFS_BTREE_CRC_BLOCKS (1<<3) /* uses extended btree blocks */ #define XFS_BTREE_OVERLAPPING (1<<4) /* overlapping intervals */ +/* + * The root of this btree is a fakeroot structure so that we can stage a btree + * rebuild without leaving it accessible via primary metadata. The ops struct + * is dynamically allocated and must be freed when the cursor is deleted. + */ +#define XFS_BTREE_STAGING (1<<5) #define XFS_BTREE_NOERROR 0 @@ -494,6 +511,7 @@ union xfs_btree_key *xfs_btree_high_key_from_key(struct xfs_btree_cur *cur, int xfs_btree_has_record(struct xfs_btree_cur *cur, union xfs_btree_irec *low, union xfs_btree_irec *high, bool *exists); bool xfs_btree_has_more_records(struct xfs_btree_cur *cur); +struct xfs_ifork *xfs_btree_ifork_ptr(struct xfs_btree_cur *cur); /* Does this cursor point to the last block in the given level? */ static inline bool @@ -512,4 +530,20 @@ xfs_btree_islastblock( return block->bb_u.s.bb_rightsib == cpu_to_be32(NULLAGBLOCK); } +void xfs_btree_set_ptr_null(struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr); +int xfs_btree_get_buf_block(struct xfs_btree_cur *cur, union xfs_btree_ptr *ptr, + struct xfs_btree_block **block, struct xfs_buf **bpp); +void xfs_btree_set_sibling(struct xfs_btree_cur *cur, + struct xfs_btree_block *block, union xfs_btree_ptr *ptr, + int lr); +void xfs_btree_init_block_cur(struct xfs_btree_cur *cur, + struct xfs_buf *bp, int level, int numrecs); +void xfs_btree_copy_ptrs(struct xfs_btree_cur *cur, + union xfs_btree_ptr *dst_ptr, + const union xfs_btree_ptr *src_ptr, int numptrs); +void xfs_btree_copy_keys(struct xfs_btree_cur *cur, + union xfs_btree_key *dst_key, union xfs_btree_key *src_key, + int numkeys); + #endif /* __XFS_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_btree_staging.c b/fs/xfs/libxfs/xfs_btree_staging.c new file mode 100644 index 000000000000..f464a7c7cf22 --- /dev/null +++ b/fs/xfs/libxfs/xfs_btree_staging.c @@ -0,0 +1,879 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_btree.h" +#include "xfs_trace.h" +#include "xfs_btree_staging.h" + +/* + * Staging Cursors and Fake Roots for Btrees + * ========================================= + * + * A staging btree cursor is a special type of btree cursor that callers must + * use to construct a new btree index using the btree bulk loader code. The + * bulk loading code uses the staging btree cursor to abstract the details of + * initializing new btree blocks and filling them with records or key/ptr + * pairs. Regular btree operations (e.g. queries and modifications) are not + * supported with staging cursors, and callers must not invoke them. + * + * Fake root structures contain all the information about a btree that is under + * construction by the bulk loading code. Staging btree cursors point to fake + * root structures instead of the usual AG header or inode structure. + * + * Callers are expected to initialize a fake root structure and pass it into + * the _stage_cursor function for a specific btree type. When bulk loading is + * complete, callers should call the _commit_staged_btree function for that + * specific btree type to commit the new btree into the filesystem. + */ + +/* + * Don't allow staging cursors to be duplicated because they're supposed to be + * kept private to a single thread. + */ +STATIC struct xfs_btree_cur * +xfs_btree_fakeroot_dup_cursor( + struct xfs_btree_cur *cur) +{ + ASSERT(0); + return NULL; +} + +/* + * Don't allow block allocation for a staging cursor, because staging cursors + * do not support regular btree modifications. + * + * Bulk loading uses a separate callback to obtain new blocks from a + * preallocated list, which prevents ENOSPC failures during loading. + */ +STATIC int +xfs_btree_fakeroot_alloc_block( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *start_bno, + union xfs_btree_ptr *new_bno, + int *stat) +{ + ASSERT(0); + return -EFSCORRUPTED; +} + +/* + * Don't allow block freeing for a staging cursor, because staging cursors + * do not support regular btree modifications. + */ +STATIC int +xfs_btree_fakeroot_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + ASSERT(0); + return -EFSCORRUPTED; +} + +/* Initialize a pointer to the root block from the fakeroot. */ +STATIC void +xfs_btree_fakeroot_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + struct xbtree_afakeroot *afake; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + afake = cur->bc_ag.afake; + ptr->s = cpu_to_be32(afake->af_root); +} + +/* + * Bulk Loading for AG Btrees + * ========================== + * + * For a btree rooted in an AG header, pass a xbtree_afakeroot structure to the + * staging cursor. Callers should initialize this to zero. + * + * The _stage_cursor() function for a specific btree type should call + * xfs_btree_stage_afakeroot to set up the in-memory cursor as a staging + * cursor. The corresponding _commit_staged_btree() function should log the + * new root and call xfs_btree_commit_afakeroot() to transform the staging + * cursor into a regular btree cursor. + */ + +/* Update the btree root information for a per-AG fake root. */ +STATIC void +xfs_btree_afakeroot_set_root( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + int inc) +{ + struct xbtree_afakeroot *afake = cur->bc_ag.afake; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + afake->af_root = be32_to_cpu(ptr->s); + afake->af_levels += inc; +} + +/* + * Initialize a AG-rooted btree cursor with the given AG btree fake root. + * The btree cursor's bc_ops will be overridden as needed to make the staging + * functionality work. + */ +void +xfs_btree_stage_afakeroot( + struct xfs_btree_cur *cur, + struct xbtree_afakeroot *afake) +{ + struct xfs_btree_ops *nops; + + ASSERT(!(cur->bc_flags & XFS_BTREE_STAGING)); + ASSERT(!(cur->bc_flags & XFS_BTREE_ROOT_IN_INODE)); + ASSERT(cur->bc_tp == NULL); + + nops = kmem_alloc(sizeof(struct xfs_btree_ops), KM_NOFS); + memcpy(nops, cur->bc_ops, sizeof(struct xfs_btree_ops)); + nops->alloc_block = xfs_btree_fakeroot_alloc_block; + nops->free_block = xfs_btree_fakeroot_free_block; + nops->init_ptr_from_cur = xfs_btree_fakeroot_init_ptr_from_cur; + nops->set_root = xfs_btree_afakeroot_set_root; + nops->dup_cursor = xfs_btree_fakeroot_dup_cursor; + + cur->bc_ag.afake = afake; + cur->bc_nlevels = afake->af_levels; + cur->bc_ops = nops; + cur->bc_flags |= XFS_BTREE_STAGING; +} + +/* + * Transform an AG-rooted staging btree cursor back into a regular cursor by + * substituting a real btree root for the fake one and restoring normal btree + * cursor ops. The caller must log the btree root change prior to calling + * this. + */ +void +xfs_btree_commit_afakeroot( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp, + const struct xfs_btree_ops *ops) +{ + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + ASSERT(cur->bc_tp == NULL); + + trace_xfs_btree_commit_afakeroot(cur); + + kmem_free((void *)cur->bc_ops); + cur->bc_ag.agbp = agbp; + cur->bc_ops = ops; + cur->bc_flags &= ~XFS_BTREE_STAGING; + cur->bc_tp = tp; +} + +/* + * Bulk Loading for Inode-Rooted Btrees + * ==================================== + * + * For a btree rooted in an inode fork, pass a xbtree_ifakeroot structure to + * the staging cursor. This structure should be initialized as follows: + * + * - if_fork_size field should be set to the number of bytes available to the + * fork in the inode. + * + * - if_fork should point to a freshly allocated struct xfs_ifork. + * + * - if_format should be set to the appropriate fork type (e.g. + * XFS_DINODE_FMT_BTREE). + * + * All other fields must be zero. + * + * The _stage_cursor() function for a specific btree type should call + * xfs_btree_stage_ifakeroot to set up the in-memory cursor as a staging + * cursor. The corresponding _commit_staged_btree() function should log the + * new root and call xfs_btree_commit_ifakeroot() to transform the staging + * cursor into a regular btree cursor. + */ + +/* + * Initialize an inode-rooted btree cursor with the given inode btree fake + * root. The btree cursor's bc_ops will be overridden as needed to make the + * staging functionality work. If new_ops is not NULL, these new ops will be + * passed out to the caller for further overriding. + */ +void +xfs_btree_stage_ifakeroot( + struct xfs_btree_cur *cur, + struct xbtree_ifakeroot *ifake, + struct xfs_btree_ops **new_ops) +{ + struct xfs_btree_ops *nops; + + ASSERT(!(cur->bc_flags & XFS_BTREE_STAGING)); + ASSERT(cur->bc_flags & XFS_BTREE_ROOT_IN_INODE); + ASSERT(cur->bc_tp == NULL); + + nops = kmem_alloc(sizeof(struct xfs_btree_ops), KM_NOFS); + memcpy(nops, cur->bc_ops, sizeof(struct xfs_btree_ops)); + nops->alloc_block = xfs_btree_fakeroot_alloc_block; + nops->free_block = xfs_btree_fakeroot_free_block; + nops->init_ptr_from_cur = xfs_btree_fakeroot_init_ptr_from_cur; + nops->dup_cursor = xfs_btree_fakeroot_dup_cursor; + + cur->bc_ino.ifake = ifake; + cur->bc_nlevels = ifake->if_levels; + cur->bc_ops = nops; + cur->bc_flags |= XFS_BTREE_STAGING; + + if (new_ops) + *new_ops = nops; +} + +/* + * Transform an inode-rooted staging btree cursor back into a regular cursor by + * substituting a real btree root for the fake one and restoring normal btree + * cursor ops. The caller must log the btree root change prior to calling + * this. + */ +void +xfs_btree_commit_ifakeroot( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + int whichfork, + const struct xfs_btree_ops *ops) +{ + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + ASSERT(cur->bc_tp == NULL); + + trace_xfs_btree_commit_ifakeroot(cur); + + kmem_free((void *)cur->bc_ops); + cur->bc_ino.ifake = NULL; + cur->bc_ino.whichfork = whichfork; + cur->bc_ops = ops; + cur->bc_flags &= ~XFS_BTREE_STAGING; + cur->bc_tp = tp; +} + +/* + * Bulk Loading of Staged Btrees + * ============================= + * + * This interface is used with a staged btree cursor to create a totally new + * btree with a large number of records (i.e. more than what would fit in a + * single root block). When the creation is complete, the new root can be + * linked atomically into the filesystem by committing the staged cursor. + * + * Creation of a new btree proceeds roughly as follows: + * + * The first step is to initialize an appropriate fake btree root structure and + * then construct a staged btree cursor. Refer to the block comments about + * "Bulk Loading for AG Btrees" and "Bulk Loading for Inode-Rooted Btrees" for + * more information about how to do this. + * + * The second step is to initialize a struct xfs_btree_bload context as + * documented in the structure definition. + * + * The third step is to call xfs_btree_bload_compute_geometry to compute the + * height of and the number of blocks needed to construct the btree. See the + * section "Computing the Geometry of the New Btree" for details about this + * computation. + * + * In step four, the caller must allocate xfs_btree_bload.nr_blocks blocks and + * save them for later use by ->claim_block(). Bulk loading requires all + * blocks to be allocated beforehand to avoid ENOSPC failures midway through a + * rebuild, and to minimize seek distances of the new btree. + * + * Step five is to call xfs_btree_bload() to start constructing the btree. + * + * The final step is to commit the staging btree cursor, which logs the new + * btree root and turns the staging cursor into a regular cursor. The caller + * is responsible for cleaning up the previous btree blocks, if any. + * + * Computing the Geometry of the New Btree + * ======================================= + * + * The number of items placed in each btree block is computed via the following + * algorithm: For leaf levels, the number of items for the level is nr_records + * in the bload structure. For node levels, the number of items for the level + * is the number of blocks in the next lower level of the tree. For each + * level, the desired number of items per block is defined as: + * + * desired = max(minrecs, maxrecs - slack factor) + * + * The number of blocks for the level is defined to be: + * + * blocks = floor(nr_items / desired) + * + * Note this is rounded down so that the npb calculation below will never fall + * below minrecs. The number of items that will actually be loaded into each + * btree block is defined as: + * + * npb = nr_items / blocks + * + * Some of the leftmost blocks in the level will contain one extra record as + * needed to handle uneven division. If the number of records in any block + * would exceed maxrecs for that level, blocks is incremented and npb is + * recalculated. + * + * In other words, we compute the number of blocks needed to satisfy a given + * loading level, then spread the items as evenly as possible. + * + * The height and number of fs blocks required to create the btree are computed + * and returned via btree_height and nr_blocks. + */ + +/* + * Put a btree block that we're loading onto the ordered list and release it. + * The btree blocks will be written to disk when bulk loading is finished. + */ +static void +xfs_btree_bload_drop_buf( + struct list_head *buffers_list, + struct xfs_buf **bpp) +{ + if (*bpp == NULL) + return; + + if (!xfs_buf_delwri_queue(*bpp, buffers_list)) + ASSERT(0); + + xfs_buf_relse(*bpp); + *bpp = NULL; +} + +/* + * Allocate and initialize one btree block for bulk loading. + * + * The new btree block will have its level and numrecs fields set to the values + * of the level and nr_this_block parameters, respectively. + * + * The caller should ensure that ptrp, bpp, and blockp refer to the left + * sibling of the new block, if there is any. On exit, ptrp, bpp, and blockp + * will all point to the new block. + */ +STATIC int +xfs_btree_bload_prep_block( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + struct list_head *buffers_list, + unsigned int level, + unsigned int nr_this_block, + union xfs_btree_ptr *ptrp, /* in/out */ + struct xfs_buf **bpp, /* in/out */ + struct xfs_btree_block **blockp, /* in/out */ + void *priv) +{ + union xfs_btree_ptr new_ptr; + struct xfs_buf *new_bp; + struct xfs_btree_block *new_block; + int ret; + + if ((cur->bc_flags & XFS_BTREE_ROOT_IN_INODE) && + level == cur->bc_nlevels - 1) { + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + size_t new_size; + + ASSERT(*bpp == NULL); + + /* Allocate a new incore btree root block. */ + new_size = bbl->iroot_size(cur, nr_this_block, priv); + ifp->if_broot = kmem_zalloc(new_size, 0); + ifp->if_broot_bytes = (int)new_size; + ifp->if_flags |= XFS_IFBROOT; + + /* Initialize it and send it out. */ + xfs_btree_init_block_int(cur->bc_mp, ifp->if_broot, + XFS_BUF_DADDR_NULL, cur->bc_btnum, level, + nr_this_block, cur->bc_ino.ip->i_ino, + cur->bc_flags); + + *bpp = NULL; + *blockp = ifp->if_broot; + xfs_btree_set_ptr_null(cur, ptrp); + return 0; + } + + /* Claim one of the caller's preallocated blocks. */ + xfs_btree_set_ptr_null(cur, &new_ptr); + ret = bbl->claim_block(cur, &new_ptr, priv); + if (ret) + return ret; + + ASSERT(!xfs_btree_ptr_is_null(cur, &new_ptr)); + + ret = xfs_btree_get_buf_block(cur, &new_ptr, &new_block, &new_bp); + if (ret) + return ret; + + /* + * The previous block (if any) is the left sibling of the new block, + * so set its right sibling pointer to the new block and drop it. + */ + if (*blockp) + xfs_btree_set_sibling(cur, *blockp, &new_ptr, XFS_BB_RIGHTSIB); + xfs_btree_bload_drop_buf(buffers_list, bpp); + + /* Initialize the new btree block. */ + xfs_btree_init_block_cur(cur, new_bp, level, nr_this_block); + xfs_btree_set_sibling(cur, new_block, ptrp, XFS_BB_LEFTSIB); + + /* Set the out parameters. */ + *bpp = new_bp; + *blockp = new_block; + xfs_btree_copy_ptrs(cur, ptrp, &new_ptr, 1); + return 0; +} + +/* Load one leaf block. */ +STATIC int +xfs_btree_bload_leaf( + struct xfs_btree_cur *cur, + unsigned int recs_this_block, + xfs_btree_bload_get_record_fn get_record, + struct xfs_btree_block *block, + void *priv) +{ + unsigned int j; + int ret; + + /* Fill the leaf block with records. */ + for (j = 1; j <= recs_this_block; j++) { + union xfs_btree_rec *block_rec; + + ret = get_record(cur, priv); + if (ret) + return ret; + block_rec = xfs_btree_rec_addr(cur, j, block); + cur->bc_ops->init_rec_from_cur(cur, block_rec); + } + + return 0; +} + +/* + * Load one node block with key/ptr pairs. + * + * child_ptr must point to a block within the next level down in the tree. A + * key/ptr entry will be created in the new node block to the block pointed to + * by child_ptr. On exit, child_ptr points to the next block on the child + * level that needs processing. + */ +STATIC int +xfs_btree_bload_node( + struct xfs_btree_cur *cur, + unsigned int recs_this_block, + union xfs_btree_ptr *child_ptr, + struct xfs_btree_block *block) +{ + unsigned int j; + int ret; + + /* Fill the node block with keys and pointers. */ + for (j = 1; j <= recs_this_block; j++) { + union xfs_btree_key child_key; + union xfs_btree_ptr *block_ptr; + union xfs_btree_key *block_key; + struct xfs_btree_block *child_block; + struct xfs_buf *child_bp; + + ASSERT(!xfs_btree_ptr_is_null(cur, child_ptr)); + + ret = xfs_btree_get_buf_block(cur, child_ptr, &child_block, + &child_bp); + if (ret) + return ret; + + block_ptr = xfs_btree_ptr_addr(cur, j, block); + xfs_btree_copy_ptrs(cur, block_ptr, child_ptr, 1); + + block_key = xfs_btree_key_addr(cur, j, block); + xfs_btree_get_keys(cur, child_block, &child_key); + xfs_btree_copy_keys(cur, block_key, &child_key, 1); + + xfs_btree_get_sibling(cur, child_block, child_ptr, + XFS_BB_RIGHTSIB); + xfs_buf_relse(child_bp); + } + + return 0; +} + +/* + * Compute the maximum number of records (or keyptrs) per block that we want to + * install at this level in the btree. Caller is responsible for having set + * @cur->bc_ino.forksize to the desired fork size, if appropriate. + */ +STATIC unsigned int +xfs_btree_bload_max_npb( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + unsigned int level) +{ + unsigned int ret; + + if (level == cur->bc_nlevels - 1 && cur->bc_ops->get_dmaxrecs) + return cur->bc_ops->get_dmaxrecs(cur, level); + + ret = cur->bc_ops->get_maxrecs(cur, level); + if (level == 0) + ret -= bbl->leaf_slack; + else + ret -= bbl->node_slack; + return ret; +} + +/* + * Compute the desired number of records (or keyptrs) per block that we want to + * install at this level in the btree, which must be somewhere between minrecs + * and max_npb. The caller is free to install fewer records per block. + */ +STATIC unsigned int +xfs_btree_bload_desired_npb( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + unsigned int level) +{ + unsigned int npb = xfs_btree_bload_max_npb(cur, bbl, level); + + /* Root blocks are not subject to minrecs rules. */ + if (level == cur->bc_nlevels - 1) + return max(1U, npb); + + return max_t(unsigned int, cur->bc_ops->get_minrecs(cur, level), npb); +} + +/* + * Compute the number of records to be stored in each block at this level and + * the number of blocks for this level. For leaf levels, we must populate an + * empty root block even if there are no records, so we have to have at least + * one block. + */ +STATIC void +xfs_btree_bload_level_geometry( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + unsigned int level, + uint64_t nr_this_level, + unsigned int *avg_per_block, + uint64_t *blocks, + uint64_t *blocks_with_extra) +{ + uint64_t npb; + uint64_t dontcare; + unsigned int desired_npb; + unsigned int maxnr; + + maxnr = cur->bc_ops->get_maxrecs(cur, level); + + /* + * Compute the number of blocks we need to fill each block with the + * desired number of records/keyptrs per block. Because desired_npb + * could be minrecs, we use regular integer division (which rounds + * the block count down) so that in the next step the effective # of + * items per block will never be less than desired_npb. + */ + desired_npb = xfs_btree_bload_desired_npb(cur, bbl, level); + *blocks = div64_u64_rem(nr_this_level, desired_npb, &dontcare); + *blocks = max(1ULL, *blocks); + + /* + * Compute the number of records that we will actually put in each + * block, assuming that we want to spread the records evenly between + * the blocks. Take care that the effective # of items per block (npb) + * won't exceed maxrecs even for the blocks that get an extra record, + * since desired_npb could be maxrecs, and in the previous step we + * rounded the block count down. + */ + npb = div64_u64_rem(nr_this_level, *blocks, blocks_with_extra); + if (npb > maxnr || (npb == maxnr && *blocks_with_extra > 0)) { + (*blocks)++; + npb = div64_u64_rem(nr_this_level, *blocks, blocks_with_extra); + } + + *avg_per_block = min_t(uint64_t, npb, nr_this_level); + + trace_xfs_btree_bload_level_geometry(cur, level, nr_this_level, + *avg_per_block, desired_npb, *blocks, + *blocks_with_extra); +} + +/* + * Ensure a slack value is appropriate for the btree. + * + * If the slack value is negative, set slack so that we fill the block to + * halfway between minrecs and maxrecs. Make sure the slack is never so large + * that we can underflow minrecs. + */ +static void +xfs_btree_bload_ensure_slack( + struct xfs_btree_cur *cur, + int *slack, + int level) +{ + int maxr; + int minr; + + maxr = cur->bc_ops->get_maxrecs(cur, level); + minr = cur->bc_ops->get_minrecs(cur, level); + + /* + * If slack is negative, automatically set slack so that we load the + * btree block approximately halfway between minrecs and maxrecs. + * Generally, this will net us 75% loading. + */ + if (*slack < 0) + *slack = maxr - ((maxr + minr) >> 1); + + *slack = min(*slack, maxr - minr); +} + +/* + * Prepare a btree cursor for a bulk load operation by computing the geometry + * fields in bbl. Caller must ensure that the btree cursor is a staging + * cursor. This function can be called multiple times. + */ +int +xfs_btree_bload_compute_geometry( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + uint64_t nr_records) +{ + uint64_t nr_blocks = 0; + uint64_t nr_this_level; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + /* + * Make sure that the slack values make sense for traditional leaf and + * node blocks. Inode-rooted btrees will return different minrecs and + * maxrecs values for the root block (bc_nlevels == level - 1). We're + * checking levels 0 and 1 here, so set bc_nlevels such that the btree + * code doesn't interpret either as the root level. + */ + cur->bc_nlevels = XFS_BTREE_MAXLEVELS - 1; + xfs_btree_bload_ensure_slack(cur, &bbl->leaf_slack, 0); + xfs_btree_bload_ensure_slack(cur, &bbl->node_slack, 1); + + bbl->nr_records = nr_this_level = nr_records; + for (cur->bc_nlevels = 1; cur->bc_nlevels < XFS_BTREE_MAXLEVELS;) { + uint64_t level_blocks; + uint64_t dontcare64; + unsigned int level = cur->bc_nlevels - 1; + unsigned int avg_per_block; + + xfs_btree_bload_level_geometry(cur, bbl, level, nr_this_level, + &avg_per_block, &level_blocks, &dontcare64); + + if (cur->bc_flags & XFS_BTREE_ROOT_IN_INODE) { + /* + * If all the items we want to store at this level + * would fit in the inode root block, then we have our + * btree root and are done. + * + * Note that bmap btrees forbid records in the root. + */ + if (level != 0 && nr_this_level <= avg_per_block) { + nr_blocks++; + break; + } + + /* + * Otherwise, we have to store all the items for this + * level in traditional btree blocks and therefore need + * another level of btree to point to those blocks. + * + * We have to re-compute the geometry for each level of + * an inode-rooted btree because the geometry differs + * between a btree root in an inode fork and a + * traditional btree block. + * + * This distinction is made in the btree code based on + * whether level == bc_nlevels - 1. Based on the + * previous root block size check against the root + * block geometry, we know that we aren't yet ready to + * populate the root. Increment bc_nevels and + * recalculate the geometry for a traditional + * block-based btree level. + */ + cur->bc_nlevels++; + xfs_btree_bload_level_geometry(cur, bbl, level, + nr_this_level, &avg_per_block, + &level_blocks, &dontcare64); + } else { + /* + * If all the items we want to store at this level + * would fit in a single root block, we're done. + */ + if (nr_this_level <= avg_per_block) { + nr_blocks++; + break; + } + + /* Otherwise, we need another level of btree. */ + cur->bc_nlevels++; + } + + nr_blocks += level_blocks; + nr_this_level = level_blocks; + } + + if (cur->bc_nlevels == XFS_BTREE_MAXLEVELS) + return -EOVERFLOW; + + bbl->btree_height = cur->bc_nlevels; + if (cur->bc_flags & XFS_BTREE_ROOT_IN_INODE) + bbl->nr_blocks = nr_blocks - 1; + else + bbl->nr_blocks = nr_blocks; + return 0; +} + +/* Bulk load a btree given the parameters and geometry established in bbl. */ +int +xfs_btree_bload( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + void *priv) +{ + struct list_head buffers_list; + union xfs_btree_ptr child_ptr; + union xfs_btree_ptr ptr; + struct xfs_buf *bp = NULL; + struct xfs_btree_block *block = NULL; + uint64_t nr_this_level = bbl->nr_records; + uint64_t blocks; + uint64_t i; + uint64_t blocks_with_extra; + uint64_t total_blocks = 0; + unsigned int avg_per_block; + unsigned int level = 0; + int ret; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + INIT_LIST_HEAD(&buffers_list); + cur->bc_nlevels = bbl->btree_height; + xfs_btree_set_ptr_null(cur, &child_ptr); + xfs_btree_set_ptr_null(cur, &ptr); + + xfs_btree_bload_level_geometry(cur, bbl, level, nr_this_level, + &avg_per_block, &blocks, &blocks_with_extra); + + /* Load each leaf block. */ + for (i = 0; i < blocks; i++) { + unsigned int nr_this_block = avg_per_block; + + /* + * Due to rounding, btree blocks will not be evenly populated + * in most cases. blocks_with_extra tells us how many blocks + * will receive an extra record to distribute the excess across + * the current level as evenly as possible. + */ + if (i < blocks_with_extra) + nr_this_block++; + + ret = xfs_btree_bload_prep_block(cur, bbl, &buffers_list, level, + nr_this_block, &ptr, &bp, &block, priv); + if (ret) + goto out; + + trace_xfs_btree_bload_block(cur, level, i, blocks, &ptr, + nr_this_block); + + ret = xfs_btree_bload_leaf(cur, nr_this_block, bbl->get_record, + block, priv); + if (ret) + goto out; + + /* + * Record the leftmost leaf pointer so we know where to start + * with the first node level. + */ + if (i == 0) + xfs_btree_copy_ptrs(cur, &child_ptr, &ptr, 1); + } + total_blocks += blocks; + xfs_btree_bload_drop_buf(&buffers_list, &bp); + + /* Populate the internal btree nodes. */ + for (level = 1; level < cur->bc_nlevels; level++) { + union xfs_btree_ptr first_ptr; + + nr_this_level = blocks; + block = NULL; + xfs_btree_set_ptr_null(cur, &ptr); + + xfs_btree_bload_level_geometry(cur, bbl, level, nr_this_level, + &avg_per_block, &blocks, &blocks_with_extra); + + /* Load each node block. */ + for (i = 0; i < blocks; i++) { + unsigned int nr_this_block = avg_per_block; + + if (i < blocks_with_extra) + nr_this_block++; + + ret = xfs_btree_bload_prep_block(cur, bbl, + &buffers_list, level, nr_this_block, + &ptr, &bp, &block, priv); + if (ret) + goto out; + + trace_xfs_btree_bload_block(cur, level, i, blocks, + &ptr, nr_this_block); + + ret = xfs_btree_bload_node(cur, nr_this_block, + &child_ptr, block); + if (ret) + goto out; + + /* + * Record the leftmost node pointer so that we know + * where to start the next node level above this one. + */ + if (i == 0) + xfs_btree_copy_ptrs(cur, &first_ptr, &ptr, 1); + } + total_blocks += blocks; + xfs_btree_bload_drop_buf(&buffers_list, &bp); + xfs_btree_copy_ptrs(cur, &child_ptr, &first_ptr, 1); + } + + /* Initialize the new root. */ + if (cur->bc_flags & XFS_BTREE_ROOT_IN_INODE) { + ASSERT(xfs_btree_ptr_is_null(cur, &ptr)); + cur->bc_ino.ifake->if_levels = cur->bc_nlevels; + cur->bc_ino.ifake->if_blocks = total_blocks - 1; + } else { + cur->bc_ag.afake->af_root = be32_to_cpu(ptr.s); + cur->bc_ag.afake->af_levels = cur->bc_nlevels; + cur->bc_ag.afake->af_blocks = total_blocks; + } + + /* + * Write the new blocks to disk. If the ordered list isn't empty after + * that, then something went wrong and we have to fail. This should + * never happen, but we'll check anyway. + */ + ret = xfs_buf_delwri_submit(&buffers_list); + if (ret) + goto out; + if (!list_empty(&buffers_list)) { + ASSERT(list_empty(&buffers_list)); + ret = -EIO; + } + +out: + xfs_buf_delwri_cancel(&buffers_list); + if (bp) + xfs_buf_relse(bp); + return ret; +} diff --git a/fs/xfs/libxfs/xfs_btree_staging.h b/fs/xfs/libxfs/xfs_btree_staging.h new file mode 100644 index 000000000000..643f0f9b2994 --- /dev/null +++ b/fs/xfs/libxfs/xfs_btree_staging.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_BTREE_STAGING_H__ +#define __XFS_BTREE_STAGING_H__ + +/* Fake root for an AG-rooted btree. */ +struct xbtree_afakeroot { + /* AG block number of the new btree root. */ + xfs_agblock_t af_root; + + /* Height of the new btree. */ + unsigned int af_levels; + + /* Number of blocks used by the btree. */ + unsigned int af_blocks; +}; + +/* Cursor interactions with with fake roots for AG-rooted btrees. */ +void xfs_btree_stage_afakeroot(struct xfs_btree_cur *cur, + struct xbtree_afakeroot *afake); +void xfs_btree_commit_afakeroot(struct xfs_btree_cur *cur, struct xfs_trans *tp, + struct xfs_buf *agbp, const struct xfs_btree_ops *ops); + +/* Fake root for an inode-rooted btree. */ +struct xbtree_ifakeroot { + /* Fake inode fork. */ + struct xfs_ifork *if_fork; + + /* Number of blocks used by the btree. */ + int64_t if_blocks; + + /* Height of the new btree. */ + unsigned int if_levels; + + /* Number of bytes available for this fork in the inode. */ + unsigned int if_fork_size; + + /* Fork format. */ + unsigned int if_format; + + /* Number of records. */ + unsigned int if_extents; +}; + +/* Cursor interactions with with fake roots for inode-rooted btrees. */ +void xfs_btree_stage_ifakeroot(struct xfs_btree_cur *cur, + struct xbtree_ifakeroot *ifake, + struct xfs_btree_ops **new_ops); +void xfs_btree_commit_ifakeroot(struct xfs_btree_cur *cur, struct xfs_trans *tp, + int whichfork, const struct xfs_btree_ops *ops); + +/* Bulk loading of staged btrees. */ +typedef int (*xfs_btree_bload_get_record_fn)(struct xfs_btree_cur *cur, void *priv); +typedef int (*xfs_btree_bload_claim_block_fn)(struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, void *priv); +typedef size_t (*xfs_btree_bload_iroot_size_fn)(struct xfs_btree_cur *cur, + unsigned int nr_this_level, void *priv); + +struct xfs_btree_bload { + /* + * This function will be called nr_records times to load records into + * the btree. The function does this by setting the cursor's bc_rec + * field in in-core format. Records must be returned in sort order. + */ + xfs_btree_bload_get_record_fn get_record; + + /* + * This function will be called nr_blocks times to obtain a pointer + * to a new btree block on disk. Callers must preallocate all space + * for the new btree before calling xfs_btree_bload, and this function + * is what claims that reservation. + */ + xfs_btree_bload_claim_block_fn claim_block; + + /* + * This function should return the size of the in-core btree root + * block. It is only necessary for XFS_BTREE_ROOT_IN_INODE btree + * types. + */ + xfs_btree_bload_iroot_size_fn iroot_size; + + /* + * The caller should set this to the number of records that will be + * stored in the new btree. + */ + uint64_t nr_records; + + /* + * Number of free records to leave in each leaf block. If the caller + * sets this to -1, the slack value will be calculated to be be halfway + * between maxrecs and minrecs. This typically leaves the block 75% + * full. Note that slack values are not enforced on inode root blocks. + */ + int leaf_slack; + + /* + * Number of free key/ptrs pairs to leave in each node block. This + * field has the same semantics as leaf_slack. + */ + int node_slack; + + /* + * The xfs_btree_bload_compute_geometry function will set this to the + * number of btree blocks needed to store nr_records records. + */ + uint64_t nr_blocks; + + /* + * The xfs_btree_bload_compute_geometry function will set this to the + * height of the new btree. + */ + unsigned int btree_height; +}; + +int xfs_btree_bload_compute_geometry(struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, uint64_t nr_records); +int xfs_btree_bload(struct xfs_btree_cur *cur, struct xfs_btree_bload *bbl, + void *priv); + +#endif /* __XFS_BTREE_STAGING_H__ */ diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index 875e04f82541..897749c41f36 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -590,7 +590,7 @@ xfs_da3_split( node = oldblk->bp->b_addr; if (node->hdr.info.forw) { if (be32_to_cpu(node->hdr.info.forw) != addblk->blkno) { - xfs_buf_corruption_error(oldblk->bp); + xfs_buf_mark_corrupt(oldblk->bp); error = -EFSCORRUPTED; goto out; } @@ -603,7 +603,7 @@ xfs_da3_split( node = oldblk->bp->b_addr; if (node->hdr.info.back) { if (be32_to_cpu(node->hdr.info.back) != addblk->blkno) { - xfs_buf_corruption_error(oldblk->bp); + xfs_buf_mark_corrupt(oldblk->bp); error = -EFSCORRUPTED; goto out; } @@ -1624,7 +1624,7 @@ xfs_da3_node_lookup_int( } if (magic != XFS_DA_NODE_MAGIC && magic != XFS_DA3_NODE_MAGIC) { - xfs_buf_corruption_error(blk->bp); + xfs_buf_mark_corrupt(blk->bp); return -EFSCORRUPTED; } @@ -1639,7 +1639,7 @@ xfs_da3_node_lookup_int( /* Tree taller than we can handle; bail out! */ if (nodehdr.level >= XFS_DA_NODE_MAXDEPTH) { - xfs_buf_corruption_error(blk->bp); + xfs_buf_mark_corrupt(blk->bp); return -EFSCORRUPTED; } @@ -1647,7 +1647,7 @@ xfs_da3_node_lookup_int( if (blkno == args->geo->leafblk) expected_level = nodehdr.level - 1; else if (expected_level != nodehdr.level) { - xfs_buf_corruption_error(blk->bp); + xfs_buf_mark_corrupt(blk->bp); return -EFSCORRUPTED; } else expected_level--; @@ -1986,7 +1986,8 @@ xfs_da3_path_shift( ASSERT(path != NULL); ASSERT((path->active > 0) && (path->active < XFS_DA_NODE_MAXDEPTH)); level = (path->active-1) - 1; /* skip bottom layer in path */ - for (blk = &path->blk[level]; level >= 0; blk--, level--) { + for (; level >= 0; level--) { + blk = &path->blk[level]; xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, blk->bp->b_addr); @@ -2520,8 +2521,10 @@ xfs_dabuf_map( */ if (nirecs > 1) { map = kmem_zalloc(nirecs * sizeof(struct xfs_buf_map), KM_NOFS); - if (!map) + if (!map) { + error = -ENOMEM; goto out_free_irecs; + } *mapp = map; } diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 0f4fbb0889ff..53e503b6f186 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -57,9 +57,10 @@ typedef struct xfs_da_args { const uint8_t *name; /* string (maybe not NULL terminated) */ int namelen; /* length of string (maybe no NULL) */ uint8_t filetype; /* filetype of inode for directories */ - uint8_t *value; /* set of bytes (maybe contain NULLs) */ + void *value; /* set of bytes (maybe contain NULLs) */ int valuelen; /* length of value */ - int flags; /* argument flags (eg: ATTR_NOCREATE) */ + unsigned int attr_filter; /* XFS_ATTR_{ROOT,SECURE,INCOMPLETE} */ + unsigned int attr_flags; /* XATTR_{CREATE,REPLACE} */ xfs_dahash_t hashval; /* hash value of name */ xfs_ino_t inumber; /* input/output inode number */ struct xfs_inode *dp; /* directory inode to manipulate */ @@ -88,8 +89,7 @@ typedef struct xfs_da_args { #define XFS_DA_OP_ADDNAME 0x0004 /* this is an add operation */ #define XFS_DA_OP_OKNOENT 0x0008 /* lookup/add op, ENOENT ok, else die */ #define XFS_DA_OP_CILOOKUP 0x0010 /* lookup to return CI name if found */ -#define XFS_DA_OP_ALLOCVAL 0x0020 /* lookup to alloc buffer if found */ -#define XFS_DA_OP_INCOMPLETE 0x0040 /* lookup INCOMPLETE attr keys */ +#define XFS_DA_OP_NOTIME 0x0020 /* don't update inode timestamps */ #define XFS_DA_OP_FLAGS \ { XFS_DA_OP_JUSTCHECK, "JUSTCHECK" }, \ @@ -97,8 +97,7 @@ typedef struct xfs_da_args { { XFS_DA_OP_ADDNAME, "ADDNAME" }, \ { XFS_DA_OP_OKNOENT, "OKNOENT" }, \ { XFS_DA_OP_CILOOKUP, "CILOOKUP" }, \ - { XFS_DA_OP_ALLOCVAL, "ALLOCVAL" }, \ - { XFS_DA_OP_INCOMPLETE, "INCOMPLETE" } + { XFS_DA_OP_NOTIME, "NOTIME" } /* * Storage for holding state during Btree searches and split/join ops. diff --git a/fs/xfs/libxfs/xfs_da_format.h b/fs/xfs/libxfs/xfs_da_format.h index 734837a9b51a..08c0a4d98b89 100644 --- a/fs/xfs/libxfs/xfs_da_format.h +++ b/fs/xfs/libxfs/xfs_da_format.h @@ -692,19 +692,7 @@ struct xfs_attr3_leafblock { #define XFS_ATTR_ROOT (1 << XFS_ATTR_ROOT_BIT) #define XFS_ATTR_SECURE (1 << XFS_ATTR_SECURE_BIT) #define XFS_ATTR_INCOMPLETE (1 << XFS_ATTR_INCOMPLETE_BIT) - -/* - * Conversion macros for converting namespace bits from argument flags - * to ondisk flags. - */ -#define XFS_ATTR_NSP_ARGS_MASK (ATTR_ROOT | ATTR_SECURE) #define XFS_ATTR_NSP_ONDISK_MASK (XFS_ATTR_ROOT | XFS_ATTR_SECURE) -#define XFS_ATTR_NSP_ONDISK(flags) ((flags) & XFS_ATTR_NSP_ONDISK_MASK) -#define XFS_ATTR_NSP_ARGS(flags) ((flags) & XFS_ATTR_NSP_ARGS_MASK) -#define XFS_ATTR_NSP_ARGS_TO_ONDISK(x) (((x) & ATTR_ROOT ? XFS_ATTR_ROOT : 0) |\ - ((x) & ATTR_SECURE ? XFS_ATTR_SECURE : 0)) -#define XFS_ATTR_NSP_ONDISK_TO_ARGS(x) (((x) & XFS_ATTR_ROOT ? ATTR_ROOT : 0) |\ - ((x) & XFS_ATTR_SECURE ? ATTR_SECURE : 0)) /* * Alignment for namelist and valuelist entries (since they are mixed diff --git a/fs/xfs/libxfs/xfs_dir2_block.c b/fs/xfs/libxfs/xfs_dir2_block.c index d6ced59b9567..1dbf2f980a26 100644 --- a/fs/xfs/libxfs/xfs_dir2_block.c +++ b/fs/xfs/libxfs/xfs_dir2_block.c @@ -114,6 +114,23 @@ const struct xfs_buf_ops xfs_dir3_block_buf_ops = { .verify_struct = xfs_dir3_block_verify, }; +static xfs_failaddr_t +xfs_dir3_block_header_check( + struct xfs_inode *dp, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = dp->i_mount; + + if (xfs_sb_version_hascrc(&mp->m_sb)) { + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + + if (be64_to_cpu(hdr3->owner) != dp->i_ino) + return __this_address; + } + + return NULL; +} + int xfs_dir3_block_read( struct xfs_trans *tp, @@ -121,12 +138,24 @@ xfs_dir3_block_read( struct xfs_buf **bpp) { struct xfs_mount *mp = dp->i_mount; + xfs_failaddr_t fa; int err; err = xfs_da_read_buf(tp, dp, mp->m_dir_geo->datablk, 0, bpp, XFS_DATA_FORK, &xfs_dir3_block_buf_ops); - if (!err && tp && *bpp) - xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_BLOCK_BUF); + if (err || !*bpp) + return err; + + /* Check things that we can't do in the verifier. */ + fa = xfs_dir3_block_header_check(dp, *bpp); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + return -EFSCORRUPTED; + } + + xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_BLOCK_BUF); return err; } diff --git a/fs/xfs/libxfs/xfs_dir2_data.c b/fs/xfs/libxfs/xfs_dir2_data.c index b9eba8213180..375b3edb2ad2 100644 --- a/fs/xfs/libxfs/xfs_dir2_data.c +++ b/fs/xfs/libxfs/xfs_dir2_data.c @@ -394,6 +394,22 @@ static const struct xfs_buf_ops xfs_dir3_data_reada_buf_ops = { .verify_write = xfs_dir3_data_write_verify, }; +static xfs_failaddr_t +xfs_dir3_data_header_check( + struct xfs_inode *dp, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = dp->i_mount; + + if (xfs_sb_version_hascrc(&mp->m_sb)) { + struct xfs_dir3_data_hdr *hdr3 = bp->b_addr; + + if (be64_to_cpu(hdr3->hdr.owner) != dp->i_ino) + return __this_address; + } + + return NULL; +} int xfs_dir3_data_read( @@ -403,12 +419,24 @@ xfs_dir3_data_read( unsigned int flags, struct xfs_buf **bpp) { + xfs_failaddr_t fa; int err; err = xfs_da_read_buf(tp, dp, bno, flags, bpp, XFS_DATA_FORK, &xfs_dir3_data_buf_ops); - if (!err && tp && *bpp) - xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_DATA_BUF); + if (err || !*bpp) + return err; + + /* Check things that we can't do in the verifier. */ + fa = xfs_dir3_data_header_check(dp, *bpp); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + return -EFSCORRUPTED; + } + + xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_DATA_BUF); return err; } diff --git a/fs/xfs/libxfs/xfs_dir2_leaf.c b/fs/xfs/libxfs/xfs_dir2_leaf.c index a131b520aac7..95d2a3f92d75 100644 --- a/fs/xfs/libxfs/xfs_dir2_leaf.c +++ b/fs/xfs/libxfs/xfs_dir2_leaf.c @@ -1383,7 +1383,7 @@ xfs_dir2_leaf_removename( ltp = xfs_dir2_leaf_tail_p(geo, leaf); bestsp = xfs_dir2_leaf_bests_p(ltp); if (be16_to_cpu(bestsp[db]) != oldbest) { - xfs_buf_corruption_error(lbp); + xfs_buf_mark_corrupt(lbp); return -EFSCORRUPTED; } /* diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c index a0cc5e240306..6ac4aad98cd7 100644 --- a/fs/xfs/libxfs/xfs_dir2_node.c +++ b/fs/xfs/libxfs/xfs_dir2_node.c @@ -194,6 +194,8 @@ xfs_dir3_free_header_check( return __this_address; if (be32_to_cpu(hdr3->nvalid) < be32_to_cpu(hdr3->nused)) return __this_address; + if (be64_to_cpu(hdr3->hdr.owner) != dp->i_ino) + return __this_address; } else { struct xfs_dir2_free_hdr *hdr = bp->b_addr; @@ -226,8 +228,9 @@ __xfs_dir3_free_read( /* Check things that we can't do in the verifier. */ fa = xfs_dir3_free_header_check(dp, fbno, *bpp); if (fa) { - xfs_verifier_error(*bpp, -EFSCORRUPTED, fa); + __xfs_buf_mark_corrupt(*bpp, fa); xfs_trans_brelse(tp, *bpp); + *bpp = NULL; return -EFSCORRUPTED; } @@ -439,7 +442,7 @@ xfs_dir2_leaf_to_node( ltp = xfs_dir2_leaf_tail_p(args->geo, leaf); if (be32_to_cpu(ltp->bestcount) > (uint)dp->i_d.di_size / args->geo->blksize) { - xfs_buf_corruption_error(lbp); + xfs_buf_mark_corrupt(lbp); return -EFSCORRUPTED; } @@ -513,7 +516,7 @@ xfs_dir2_leafn_add( * into other peoples memory */ if (index < 0) { - xfs_buf_corruption_error(bp); + xfs_buf_mark_corrupt(bp); return -EFSCORRUPTED; } @@ -800,7 +803,7 @@ xfs_dir2_leafn_lookup_for_entry( xfs_dir3_leaf_check(dp, bp); if (leafhdr.count <= 0) { - xfs_buf_corruption_error(bp); + xfs_buf_mark_corrupt(bp); return -EFSCORRUPTED; } diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index 77e9fa385980..045556e78ee2 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -497,6 +497,23 @@ static inline bool xfs_sb_version_hascrc(struct xfs_sb *sbp) return XFS_SB_VERSION_NUM(sbp) == XFS_SB_VERSION_5; } +/* + * v5 file systems support V3 inodes only, earlier file systems support + * v2 and v1 inodes. + */ +static inline bool xfs_sb_version_has_v3inode(struct xfs_sb *sbp) +{ + return XFS_SB_VERSION_NUM(sbp) == XFS_SB_VERSION_5; +} + +static inline bool xfs_dinode_good_version(struct xfs_sb *sbp, + uint8_t version) +{ + if (xfs_sb_version_has_v3inode(sbp)) + return version == 3; + return version == 1 || version == 2; +} + static inline bool xfs_sb_version_has_pquotino(struct xfs_sb *sbp) { return XFS_SB_VERSION_NUM(sbp) == XFS_SB_VERSION_5; @@ -560,7 +577,6 @@ xfs_is_quota_inode(struct xfs_sb *sbp, xfs_ino_t ino) #define XFS_SB_DADDR ((xfs_daddr_t)0) /* daddr in filesystem/ag */ #define XFS_SB_BLOCK(mp) XFS_HDR_BLOCK(mp, XFS_SB_DADDR) -#define XFS_BUF_TO_SBP(bp) ((xfs_dsb_t *)((bp)->b_addr)) #define XFS_HDR_BLOCK(mp,d) ((xfs_agblock_t)XFS_BB_TO_FSBT(mp,d)) #define XFS_DADDR_TO_FSB(mp,d) XFS_AGB_TO_FSB(mp, \ @@ -707,7 +723,6 @@ typedef struct xfs_agf { /* disk block (xfs_daddr_t) in the AG */ #define XFS_AGF_DADDR(mp) ((xfs_daddr_t)(1 << (mp)->m_sectbb_log)) #define XFS_AGF_BLOCK(mp) XFS_HDR_BLOCK(mp, XFS_AGF_DADDR(mp)) -#define XFS_BUF_TO_AGF(bp) ((xfs_agf_t *)((bp)->b_addr)) /* * Size of the unlinked inode hash table in the agi. @@ -775,7 +790,6 @@ typedef struct xfs_agi { /* disk block (xfs_daddr_t) in the AG */ #define XFS_AGI_DADDR(mp) ((xfs_daddr_t)(2 << (mp)->m_sectbb_log)) #define XFS_AGI_BLOCK(mp) XFS_HDR_BLOCK(mp, XFS_AGI_DADDR(mp)) -#define XFS_BUF_TO_AGI(bp) ((xfs_agi_t *)((bp)->b_addr)) /* * The third a.g. block contains the a.g. freelist, an array @@ -783,21 +797,15 @@ typedef struct xfs_agi { */ #define XFS_AGFL_DADDR(mp) ((xfs_daddr_t)(3 << (mp)->m_sectbb_log)) #define XFS_AGFL_BLOCK(mp) XFS_HDR_BLOCK(mp, XFS_AGFL_DADDR(mp)) -#define XFS_BUF_TO_AGFL(bp) ((xfs_agfl_t *)((bp)->b_addr)) +#define XFS_BUF_TO_AGFL(bp) ((struct xfs_agfl *)((bp)->b_addr)) -#define XFS_BUF_TO_AGFL_BNO(mp, bp) \ - (xfs_sb_version_hascrc(&((mp)->m_sb)) ? \ - &(XFS_BUF_TO_AGFL(bp)->agfl_bno[0]) : \ - (__be32 *)(bp)->b_addr) - -typedef struct xfs_agfl { +struct xfs_agfl { __be32 agfl_magicnum; __be32 agfl_seqno; uuid_t agfl_uuid; __be64 agfl_lsn; __be32 agfl_crc; - __be32 agfl_bno[]; /* actually xfs_agfl_size(mp) */ -} __attribute__((packed)) xfs_agfl_t; +} __attribute__((packed)); #define XFS_AGFL_CRC_OFF offsetof(struct xfs_agfl, agfl_crc) @@ -946,8 +954,12 @@ enum xfs_dinode_fmt { /* * Inode size for given fs. */ -#define XFS_LITINO(mp, version) \ - ((int)(((mp)->m_sb.sb_inodesize) - xfs_dinode_size(version))) +#define XFS_DINODE_SIZE(sbp) \ + (xfs_sb_version_has_v3inode(sbp) ? \ + sizeof(struct xfs_dinode) : \ + offsetof(struct xfs_dinode, di_crc)) +#define XFS_LITINO(mp) \ + ((mp)->m_sb.sb_inodesize - XFS_DINODE_SIZE(&(mp)->m_sb)) /* * Inode data & attribute fork sizes, per inode. @@ -956,13 +968,9 @@ enum xfs_dinode_fmt { #define XFS_DFORK_BOFF(dip) ((int)((dip)->di_forkoff << 3)) #define XFS_DFORK_DSIZE(dip,mp) \ - (XFS_DFORK_Q(dip) ? \ - XFS_DFORK_BOFF(dip) : \ - XFS_LITINO(mp, (dip)->di_version)) + (XFS_DFORK_Q(dip) ? XFS_DFORK_BOFF(dip) : XFS_LITINO(mp)) #define XFS_DFORK_ASIZE(dip,mp) \ - (XFS_DFORK_Q(dip) ? \ - XFS_LITINO(mp, (dip)->di_version) - XFS_DFORK_BOFF(dip) : \ - 0) + (XFS_DFORK_Q(dip) ? XFS_LITINO(mp) - XFS_DFORK_BOFF(dip) : 0) #define XFS_DFORK_SIZE(dip,mp,w) \ ((w) == XFS_DATA_FORK ? \ XFS_DFORK_DSIZE(dip, mp) : \ diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index ef95ca07d084..245188e4f6d3 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -568,10 +568,40 @@ typedef struct xfs_fsop_setdm_handlereq { struct fsdmidata __user *data; /* DMAPI data */ } xfs_fsop_setdm_handlereq_t; +/* + * Flags passed in xfs_attr_multiop.am_flags for the attr ioctl interface. + * + * NOTE: Must match the values declared in libattr without the XFS_IOC_ prefix. + */ +#define XFS_IOC_ATTR_ROOT 0x0002 /* use attrs in root namespace */ +#define XFS_IOC_ATTR_SECURE 0x0008 /* use attrs in security namespace */ +#define XFS_IOC_ATTR_CREATE 0x0010 /* fail if attr already exists */ +#define XFS_IOC_ATTR_REPLACE 0x0020 /* fail if attr does not exist */ + typedef struct xfs_attrlist_cursor { __u32 opaque[4]; } xfs_attrlist_cursor_t; +/* + * Define how lists of attribute names are returned to userspace from the + * XFS_IOC_ATTRLIST_BY_HANDLE ioctl. struct xfs_attrlist is the header at the + * beginning of the returned buffer, and a each entry in al_offset contains the + * relative offset of an xfs_attrlist_ent containing the actual entry. + * + * NOTE: struct xfs_attrlist must match struct attrlist defined in libattr, and + * struct xfs_attrlist_ent must match struct attrlist_ent defined in libattr. + */ +struct xfs_attrlist { + __s32 al_count; /* number of entries in attrlist */ + __s32 al_more; /* T/F: more attrs (do call again) */ + __s32 al_offset[1]; /* byte offsets of attrs [var-sized] */ +}; + +struct xfs_attrlist_ent { /* data from attr_list() */ + __u32 a_valuelen; /* number bytes in value of attr */ + char a_name[1]; /* attr name (NULL terminated) */ +}; + typedef struct xfs_fsop_attrlist_handlereq { struct xfs_fsop_handlereq hreq; /* handle interface structure */ struct xfs_attrlist_cursor pos; /* opaque cookie, list offset */ @@ -589,7 +619,7 @@ typedef struct xfs_attr_multiop { void __user *am_attrname; void __user *am_attrvalue; __u32 am_length; - __u32 am_flags; + __u32 am_flags; /* XFS_IOC_ATTR_* */ } xfs_attr_multiop_t; typedef struct xfs_fsop_attrmulti_handlereq { diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c index bf161e930f1d..7fcf62b324b0 100644 --- a/fs/xfs/libxfs/xfs_ialloc.c +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -105,7 +105,7 @@ xfs_inobt_get_rec( int *stat) { struct xfs_mount *mp = cur->bc_mp; - xfs_agnumber_t agno = cur->bc_private.a.agno; + xfs_agnumber_t agno = cur->bc_ag.agno; union xfs_btree_rec *rec; int error; uint64_t realfree; @@ -177,7 +177,7 @@ xfs_inobt_insert( xfs_btnum_t btnum) { struct xfs_btree_cur *cur; - struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); + struct xfs_agi *agi = agbp->b_addr; xfs_agnumber_t agno = be32_to_cpu(agi->agi_seqno); xfs_agino_t thisino; int i; @@ -304,7 +304,7 @@ xfs_ialloc_inode_init( * That means for v3 inode we log the entire buffer rather than just the * inode cores. */ - if (xfs_sb_version_hascrc(&mp->m_sb)) { + if (xfs_sb_version_has_v3inode(&mp->m_sb)) { version = 3; ino = XFS_AGINO_TO_INO(mp, agno, XFS_AGB_TO_AGINO(mp, agbno)); @@ -339,7 +339,7 @@ xfs_ialloc_inode_init( xfs_buf_zero(fbuf, 0, BBTOB(fbuf->b_length)); for (i = 0; i < M_IGEO(mp)->inodes_per_cluster; i++) { int ioffset = i << mp->m_sb.sb_inodelog; - uint isize = xfs_dinode_size(version); + uint isize = XFS_DINODE_SIZE(&mp->m_sb); free = xfs_make_iptr(mp, fbuf, i); free->di_magic = cpu_to_be16(XFS_DINODE_MAGIC); @@ -525,7 +525,7 @@ xfs_inobt_insert_sprec( bool merge) /* merge or replace */ { struct xfs_btree_cur *cur; - struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); + struct xfs_agi *agi = agbp->b_addr; xfs_agnumber_t agno = be32_to_cpu(agi->agi_seqno); int error; int i; @@ -658,7 +658,7 @@ xfs_ialloc_ag_alloc( * chunk of inodes. If the filesystem is striped, this will fill * an entire stripe unit with inodes. */ - agi = XFS_BUF_TO_AGI(agbp); + agi = agbp->b_addr; newino = be32_to_cpu(agi->agi_newino); agno = be32_to_cpu(agi->agi_seqno); args.agbno = XFS_AGINO_TO_AGBNO(args.mp, newino) + @@ -1130,7 +1130,7 @@ xfs_dialloc_ag_inobt( xfs_ino_t *inop) { struct xfs_mount *mp = tp->t_mountp; - struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); + struct xfs_agi *agi = agbp->b_addr; xfs_agnumber_t agno = be32_to_cpu(agi->agi_seqno); xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent); xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent); @@ -1583,7 +1583,7 @@ xfs_dialloc_ag( xfs_ino_t *inop) { struct xfs_mount *mp = tp->t_mountp; - struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); + struct xfs_agi *agi = agbp->b_addr; xfs_agnumber_t agno = be32_to_cpu(agi->agi_seqno); xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent); xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent); @@ -1943,7 +1943,7 @@ xfs_difree_inobt( struct xfs_icluster *xic, struct xfs_inobt_rec_incore *orec) { - struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); + struct xfs_agi *agi = agbp->b_addr; xfs_agnumber_t agno = be32_to_cpu(agi->agi_seqno); struct xfs_perag *pag; struct xfs_btree_cur *cur; @@ -2079,7 +2079,7 @@ xfs_difree_finobt( xfs_agino_t agino, struct xfs_inobt_rec_incore *ibtrec) /* inobt record */ { - struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); + struct xfs_agi *agi = agbp->b_addr; xfs_agnumber_t agno = be32_to_cpu(agi->agi_seqno); struct xfs_btree_cur *cur; struct xfs_inobt_rec_incore rec; @@ -2489,9 +2489,8 @@ xfs_ialloc_log_agi( sizeof(xfs_agi_t) }; #ifdef DEBUG - xfs_agi_t *agi; /* allocation group header */ + struct xfs_agi *agi = bp->b_addr; - agi = XFS_BUF_TO_AGI(bp); ASSERT(agi->agi_magicnum == cpu_to_be32(XFS_AGI_MAGIC)); #endif @@ -2523,14 +2522,13 @@ xfs_agi_verify( struct xfs_buf *bp) { struct xfs_mount *mp = bp->b_mount; - struct xfs_agi *agi = XFS_BUF_TO_AGI(bp); + struct xfs_agi *agi = bp->b_addr; int i; if (xfs_sb_version_hascrc(&mp->m_sb)) { if (!uuid_equal(&agi->agi_uuid, &mp->m_sb.sb_meta_uuid)) return __this_address; - if (!xfs_log_check_lsn(mp, - be64_to_cpu(XFS_BUF_TO_AGI(bp)->agi_lsn))) + if (!xfs_log_check_lsn(mp, be64_to_cpu(agi->agi_lsn))) return __this_address; } @@ -2593,6 +2591,7 @@ xfs_agi_write_verify( { struct xfs_mount *mp = bp->b_mount; struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_agi *agi = bp->b_addr; xfs_failaddr_t fa; fa = xfs_agi_verify(bp); @@ -2605,7 +2604,7 @@ xfs_agi_write_verify( return; if (bip) - XFS_BUF_TO_AGI(bp)->agi_lsn = cpu_to_be64(bip->bli_item.li_lsn); + agi->agi_lsn = cpu_to_be64(bip->bli_item.li_lsn); xfs_buf_update_cksum(bp, XFS_AGI_CRC_OFF); } @@ -2661,7 +2660,7 @@ xfs_ialloc_read_agi( if (error) return error; - agi = XFS_BUF_TO_AGI(*bpp); + agi = (*bpp)->b_addr; pag = xfs_perag_get(mp, agno); if (!pag->pagi_init) { pag->pagi_freecount = be32_to_cpu(agi->agi_freecount); @@ -2873,7 +2872,7 @@ xfs_ialloc_setup_geometry( * cannot change the behavior. */ igeo->inode_cluster_size_raw = XFS_INODE_BIG_CLUSTER_SIZE; - if (xfs_sb_version_hascrc(&mp->m_sb)) { + if (xfs_sb_version_has_v3inode(&mp->m_sb)) { int new_size = igeo->inode_cluster_size_raw; new_size *= mp->m_sb.sb_inodesize / XFS_DINODE_MIN_SIZE; diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.c b/fs/xfs/libxfs/xfs_ialloc_btree.c index b82992f795aa..b2c122ad8f0e 100644 --- a/fs/xfs/libxfs/xfs_ialloc_btree.c +++ b/fs/xfs/libxfs/xfs_ialloc_btree.c @@ -12,6 +12,7 @@ #include "xfs_bit.h" #include "xfs_mount.h" #include "xfs_btree.h" +#include "xfs_btree_staging.h" #include "xfs_ialloc.h" #include "xfs_ialloc_btree.h" #include "xfs_alloc.h" @@ -20,7 +21,6 @@ #include "xfs_trans.h" #include "xfs_rmap.h" - STATIC int xfs_inobt_get_minrecs( struct xfs_btree_cur *cur, @@ -34,7 +34,7 @@ xfs_inobt_dup_cursor( struct xfs_btree_cur *cur) { return xfs_inobt_init_cursor(cur->bc_mp, cur->bc_tp, - cur->bc_private.a.agbp, cur->bc_private.a.agno, + cur->bc_ag.agbp, cur->bc_ag.agno, cur->bc_btnum); } @@ -44,8 +44,8 @@ xfs_inobt_set_root( union xfs_btree_ptr *nptr, int inc) /* level change */ { - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agi *agi = agbp->b_addr; agi->agi_root = nptr->s; be32_add_cpu(&agi->agi_level, inc); @@ -58,8 +58,8 @@ xfs_finobt_set_root( union xfs_btree_ptr *nptr, int inc) /* level change */ { - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agi *agi = agbp->b_addr; agi->agi_free_root = nptr->s; be32_add_cpu(&agi->agi_free_level, inc); @@ -83,7 +83,7 @@ __xfs_inobt_alloc_block( args.tp = cur->bc_tp; args.mp = cur->bc_mp; args.oinfo = XFS_RMAP_OINFO_INOBT; - args.fsbno = XFS_AGB_TO_FSB(args.mp, cur->bc_private.a.agno, sbno); + args.fsbno = XFS_AGB_TO_FSB(args.mp, cur->bc_ag.agno, sbno); args.minlen = 1; args.maxlen = 1; args.prod = 1; @@ -212,9 +212,9 @@ xfs_inobt_init_ptr_from_cur( struct xfs_btree_cur *cur, union xfs_btree_ptr *ptr) { - struct xfs_agi *agi = XFS_BUF_TO_AGI(cur->bc_private.a.agbp); + struct xfs_agi *agi = cur->bc_ag.agbp->b_addr; - ASSERT(cur->bc_private.a.agno == be32_to_cpu(agi->agi_seqno)); + ASSERT(cur->bc_ag.agno == be32_to_cpu(agi->agi_seqno)); ptr->s = agi->agi_root; } @@ -224,9 +224,9 @@ xfs_finobt_init_ptr_from_cur( struct xfs_btree_cur *cur, union xfs_btree_ptr *ptr) { - struct xfs_agi *agi = XFS_BUF_TO_AGI(cur->bc_private.a.agbp); + struct xfs_agi *agi = cur->bc_ag.agbp->b_addr; - ASSERT(cur->bc_private.a.agno == be32_to_cpu(agi->agi_seqno)); + ASSERT(cur->bc_ag.agno == be32_to_cpu(agi->agi_seqno)); ptr->s = agi->agi_free_root; } @@ -400,32 +400,27 @@ static const struct xfs_btree_ops xfs_finobt_ops = { }; /* - * Allocate a new inode btree cursor. + * Initialize a new inode btree cursor. */ -struct xfs_btree_cur * /* new inode btree cursor */ -xfs_inobt_init_cursor( +static struct xfs_btree_cur * +xfs_inobt_init_common( struct xfs_mount *mp, /* file system mount point */ struct xfs_trans *tp, /* transaction pointer */ - struct xfs_buf *agbp, /* buffer for agi structure */ xfs_agnumber_t agno, /* allocation group number */ xfs_btnum_t btnum) /* ialloc or free ino btree */ { - struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); struct xfs_btree_cur *cur; cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS); - cur->bc_tp = tp; cur->bc_mp = mp; cur->bc_btnum = btnum; if (btnum == XFS_BTNUM_INO) { - cur->bc_nlevels = be32_to_cpu(agi->agi_level); - cur->bc_ops = &xfs_inobt_ops; cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_ibt_2); + cur->bc_ops = &xfs_inobt_ops; } else { - cur->bc_nlevels = be32_to_cpu(agi->agi_free_level); - cur->bc_ops = &xfs_finobt_ops; cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_fibt_2); + cur->bc_ops = &xfs_finobt_ops; } cur->bc_blocklog = mp->m_sb.sb_blocklog; @@ -433,12 +428,75 @@ xfs_inobt_init_cursor( if (xfs_sb_version_hascrc(&mp->m_sb)) cur->bc_flags |= XFS_BTREE_CRC_BLOCKS; - cur->bc_private.a.agbp = agbp; - cur->bc_private.a.agno = agno; - + cur->bc_ag.agno = agno; return cur; } +/* Create an inode btree cursor. */ +struct xfs_btree_cur * +xfs_inobt_init_cursor( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_buf *agbp, + xfs_agnumber_t agno, + xfs_btnum_t btnum) +{ + struct xfs_btree_cur *cur; + struct xfs_agi *agi = agbp->b_addr; + + cur = xfs_inobt_init_common(mp, tp, agno, btnum); + if (btnum == XFS_BTNUM_INO) + cur->bc_nlevels = be32_to_cpu(agi->agi_level); + else + cur->bc_nlevels = be32_to_cpu(agi->agi_free_level); + cur->bc_ag.agbp = agbp; + return cur; +} + +/* Create an inode btree cursor with a fake root for staging. */ +struct xfs_btree_cur * +xfs_inobt_stage_cursor( + struct xfs_mount *mp, + struct xbtree_afakeroot *afake, + xfs_agnumber_t agno, + xfs_btnum_t btnum) +{ + struct xfs_btree_cur *cur; + + cur = xfs_inobt_init_common(mp, NULL, agno, btnum); + xfs_btree_stage_afakeroot(cur, afake); + return cur; +} + +/* + * Install a new inobt btree root. Caller is responsible for invalidating + * and freeing the old btree blocks. + */ +void +xfs_inobt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_agi *agi = agbp->b_addr; + struct xbtree_afakeroot *afake = cur->bc_ag.afake; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + if (cur->bc_btnum == XFS_BTNUM_INO) { + agi->agi_root = cpu_to_be32(afake->af_root); + agi->agi_level = cpu_to_be32(afake->af_levels); + xfs_ialloc_log_agi(tp, agbp, XFS_AGI_ROOT | XFS_AGI_LEVEL); + xfs_btree_commit_afakeroot(cur, tp, agbp, &xfs_inobt_ops); + } else { + agi->agi_free_root = cpu_to_be32(afake->af_root); + agi->agi_free_level = cpu_to_be32(afake->af_levels); + xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREE_ROOT | + XFS_AGI_FREE_LEVEL); + xfs_btree_commit_afakeroot(cur, tp, agbp, &xfs_finobt_ops); + } +} + /* * Calculate number of records in an inobt btree block. */ diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.h b/fs/xfs/libxfs/xfs_ialloc_btree.h index 951305ecaae1..35bbd978c272 100644 --- a/fs/xfs/libxfs/xfs_ialloc_btree.h +++ b/fs/xfs/libxfs/xfs_ialloc_btree.h @@ -48,6 +48,9 @@ struct xfs_mount; extern struct xfs_btree_cur *xfs_inobt_init_cursor(struct xfs_mount *, struct xfs_trans *, struct xfs_buf *, xfs_agnumber_t, xfs_btnum_t); +struct xfs_btree_cur *xfs_inobt_stage_cursor(struct xfs_mount *mp, + struct xbtree_afakeroot *afake, xfs_agnumber_t agno, + xfs_btnum_t btnum); extern int xfs_inobt_maxrecs(struct xfs_mount *, int, int); /* ir_holemask to inode allocation bitmap conversion */ @@ -68,4 +71,7 @@ int xfs_inobt_cur(struct xfs_mount *mp, struct xfs_trans *tp, xfs_agnumber_t agno, xfs_btnum_t btnum, struct xfs_btree_cur **curpp, struct xfs_buf **agi_bpp); +void xfs_inobt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp, struct xfs_buf *agbp); + #endif /* __XFS_IALLOC_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 8afacfe4be0a..39c5a6e24915 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -44,17 +44,6 @@ xfs_inobp_check( } #endif -bool -xfs_dinode_good_version( - struct xfs_mount *mp, - __u8 version) -{ - if (xfs_sb_version_hascrc(&mp->m_sb)) - return version == 3; - - return version == 1 || version == 2; -} - /* * If we are doing readahead on an inode buffer, we might be in log recovery * reading an inode allocation buffer that hasn't yet been replayed, and hence @@ -93,7 +82,7 @@ xfs_inode_buf_verify( dip = xfs_buf_offset(bp, (i << mp->m_sb.sb_inodelog)); unlinked_ino = be32_to_cpu(dip->di_next_unlinked); di_ok = xfs_verify_magic16(bp, dip->di_magic) && - xfs_dinode_good_version(mp, dip->di_version) && + xfs_dinode_good_version(&mp->m_sb, dip->di_version) && xfs_verify_agino_or_null(mp, agno, unlinked_ino); if (unlikely(XFS_TEST_ERROR(!di_ok, mp, XFS_ERRTAG_ITOBP_INOTOBP))) { @@ -205,16 +194,14 @@ xfs_inode_from_disk( struct xfs_icdinode *to = &ip->i_d; struct inode *inode = VFS_I(ip); - /* * Convert v1 inodes immediately to v2 inode format as this is the * minimum inode version format we support in the rest of the code. + * They will also be unconditionally written back to disk as v2 inodes. */ - to->di_version = from->di_version; - if (to->di_version == 1) { + if (unlikely(from->di_version == 1)) { set_nlink(inode, be16_to_cpu(from->di_onlink)); to->di_projid = 0; - to->di_version = 2; } else { set_nlink(inode, be32_to_cpu(from->di_nlink)); to->di_projid = (prid_t)be16_to_cpu(from->di_projid_hi) << 16 | @@ -222,8 +209,8 @@ xfs_inode_from_disk( } to->di_format = from->di_format; - to->di_uid = be32_to_cpu(from->di_uid); - to->di_gid = be32_to_cpu(from->di_gid); + i_uid_write(inode, be32_to_cpu(from->di_uid)); + i_gid_write(inode, be32_to_cpu(from->di_gid)); to->di_flushiter = be16_to_cpu(from->di_flushiter); /* @@ -252,7 +239,7 @@ xfs_inode_from_disk( to->di_dmstate = be16_to_cpu(from->di_dmstate); to->di_flags = be16_to_cpu(from->di_flags); - if (to->di_version == 3) { + if (xfs_sb_version_has_v3inode(&ip->i_mount->m_sb)) { inode_set_iversion_queried(inode, be64_to_cpu(from->di_changecount)); to->di_crtime.tv_sec = be32_to_cpu(from->di_crtime.t_sec); @@ -274,10 +261,9 @@ xfs_inode_to_disk( to->di_magic = cpu_to_be16(XFS_DINODE_MAGIC); to->di_onlink = 0; - to->di_version = from->di_version; to->di_format = from->di_format; - to->di_uid = cpu_to_be32(from->di_uid); - to->di_gid = cpu_to_be32(from->di_gid); + to->di_uid = cpu_to_be32(i_uid_read(inode)); + to->di_gid = cpu_to_be32(i_gid_read(inode)); to->di_projid_lo = cpu_to_be16(from->di_projid & 0xffff); to->di_projid_hi = cpu_to_be16(from->di_projid >> 16); @@ -303,7 +289,8 @@ xfs_inode_to_disk( to->di_dmstate = cpu_to_be16(from->di_dmstate); to->di_flags = cpu_to_be16(from->di_flags); - if (from->di_version == 3) { + if (xfs_sb_version_has_v3inode(&ip->i_mount->m_sb)) { + to->di_version = 3; to->di_changecount = cpu_to_be64(inode_peek_iversion(inode)); to->di_crtime.t_sec = cpu_to_be32(from->di_crtime.tv_sec); to->di_crtime.t_nsec = cpu_to_be32(from->di_crtime.tv_nsec); @@ -315,6 +302,7 @@ xfs_inode_to_disk( uuid_copy(&to->di_uuid, &ip->i_mount->m_sb.sb_meta_uuid); to->di_flushiter = 0; } else { + to->di_version = 2; to->di_flushiter = cpu_to_be16(from->di_flushiter); } } @@ -428,7 +416,7 @@ xfs_dinode_verify_forkoff( case XFS_DINODE_FMT_LOCAL: /* fall through ... */ case XFS_DINODE_FMT_EXTENTS: /* fall through ... */ case XFS_DINODE_FMT_BTREE: - if (dip->di_forkoff >= (XFS_LITINO(mp, dip->di_version) >> 3)) + if (dip->di_forkoff >= (XFS_LITINO(mp) >> 3)) return __this_address; break; default: @@ -454,7 +442,7 @@ xfs_dinode_verify( /* Verify v3 integrity information first */ if (dip->di_version >= 3) { - if (!xfs_sb_version_hascrc(&mp->m_sb)) + if (!xfs_sb_version_has_v3inode(&mp->m_sb)) return __this_address; if (!xfs_verify_cksum((char *)dip, mp->m_sb.sb_inodesize, XFS_DINODE_CRC_OFF)) @@ -629,10 +617,9 @@ xfs_iread( /* shortcut IO on inode allocation if possible */ if ((iget_flags & XFS_IGET_CREATE) && - xfs_sb_version_hascrc(&mp->m_sb) && + xfs_sb_version_has_v3inode(&mp->m_sb) && !(mp->m_flags & XFS_MOUNT_IKEEP)) { VFS_I(ip)->i_generation = prandom_u32(); - ip->i_d.di_version = 3; return 0; } @@ -674,7 +661,6 @@ xfs_iread( * Partial initialisation of the in-core inode. Just the bits * that xfs_ialloc won't overwrite or relies on being correct. */ - ip->i_d.di_version = dip->di_version; VFS_I(ip)->i_generation = be32_to_cpu(dip->di_gen); ip->i_d.di_flushiter = be16_to_cpu(dip->di_flushiter); @@ -688,7 +674,6 @@ xfs_iread( VFS_I(ip)->i_mode = 0; } - ASSERT(ip->i_d.di_version >= 2); ip->i_delayed_blks = 0; /* diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h index fd94b1078722..9b373dcf9e34 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.h +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -16,11 +16,8 @@ struct xfs_dinode; * format specific structures at the appropriate time. */ struct xfs_icdinode { - int8_t di_version; /* inode version */ int8_t di_format; /* format of di_c data */ uint16_t di_flushiter; /* incremented on flush */ - uint32_t di_uid; /* owner's user id */ - uint32_t di_gid; /* owner's group id */ uint32_t di_projid; /* owner's project id */ xfs_fsize_t di_size; /* number of bytes in file */ xfs_rfsblock_t di_nblocks; /* # of direct & btree blocks used */ @@ -61,8 +58,6 @@ void xfs_inode_from_disk(struct xfs_inode *ip, struct xfs_dinode *from); void xfs_log_dinode_to_disk(struct xfs_log_dinode *from, struct xfs_dinode *to); -bool xfs_dinode_good_version(struct xfs_mount *mp, __u8 version); - #if defined(DEBUG) void xfs_inobp_check(struct xfs_mount *, struct xfs_buf *); #else diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index ad2b9c313fd2..518c6f0ec3a6 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -183,7 +183,7 @@ xfs_iformat_local( */ if (unlikely(size > XFS_DFORK_SIZE(dip, ip->i_mount, whichfork))) { xfs_warn(ip->i_mount, - "corrupt inode %Lu (bad size %d for local fork, size = %d).", + "corrupt inode %Lu (bad size %d for local fork, size = %zd).", (unsigned long long) ip->i_ino, size, XFS_DFORK_SIZE(dip, ip->i_mount, whichfork)); xfs_inode_verifier_error(ip, -EFSCORRUPTED, diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index 500333d0101e..668ee942be22 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -46,14 +46,9 @@ struct xfs_ifork { (ip)->i_afp : \ (ip)->i_cowfp)) #define XFS_IFORK_DSIZE(ip) \ - (XFS_IFORK_Q(ip) ? \ - XFS_IFORK_BOFF(ip) : \ - XFS_LITINO((ip)->i_mount, (ip)->i_d.di_version)) + (XFS_IFORK_Q(ip) ? XFS_IFORK_BOFF(ip) : XFS_LITINO((ip)->i_mount)) #define XFS_IFORK_ASIZE(ip) \ - (XFS_IFORK_Q(ip) ? \ - XFS_LITINO((ip)->i_mount, (ip)->i_d.di_version) - \ - XFS_IFORK_BOFF(ip) : \ - 0) + (XFS_IFORK_Q(ip) ? XFS_LITINO((ip)->i_mount) - XFS_IFORK_BOFF(ip) : 0) #define XFS_IFORK_SIZE(ip,w) \ ((w) == XFS_DATA_FORK ? \ XFS_IFORK_DSIZE(ip) : \ diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 9bac0d2e56dc..e3400c9c71cd 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -424,12 +424,10 @@ struct xfs_log_dinode { /* structure must be padded to 64 bit alignment */ }; -static inline uint xfs_log_dinode_size(int version) -{ - if (version == 3) - return sizeof(struct xfs_log_dinode); - return offsetof(struct xfs_log_dinode, di_next_unlinked); -} +#define xfs_log_dinode_size(mp) \ + (xfs_sb_version_has_v3inode(&(mp)->m_sb) ? \ + sizeof(struct xfs_log_dinode) : \ + offsetof(struct xfs_log_dinode, di_next_unlinked)) /* * Buffer Log Format definitions diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 6e1665f2cb67..2076627243b0 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -46,7 +46,7 @@ xfs_refcount_lookup_le( xfs_agblock_t bno, int *stat) { - trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_private.a.agno, bno, + trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_ag.agno, bno, XFS_LOOKUP_LE); cur->bc_rec.rc.rc_startblock = bno; cur->bc_rec.rc.rc_blockcount = 0; @@ -63,7 +63,7 @@ xfs_refcount_lookup_ge( xfs_agblock_t bno, int *stat) { - trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_private.a.agno, bno, + trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_ag.agno, bno, XFS_LOOKUP_GE); cur->bc_rec.rc.rc_startblock = bno; cur->bc_rec.rc.rc_blockcount = 0; @@ -80,7 +80,7 @@ xfs_refcount_lookup_eq( xfs_agblock_t bno, int *stat) { - trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_private.a.agno, bno, + trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_ag.agno, bno, XFS_LOOKUP_LE); cur->bc_rec.rc.rc_startblock = bno; cur->bc_rec.rc.rc_blockcount = 0; @@ -108,7 +108,7 @@ xfs_refcount_get_rec( int *stat) { struct xfs_mount *mp = cur->bc_mp; - xfs_agnumber_t agno = cur->bc_private.a.agno; + xfs_agnumber_t agno = cur->bc_ag.agno; union xfs_btree_rec *rec; int error; xfs_agblock_t realstart; @@ -119,7 +119,7 @@ xfs_refcount_get_rec( xfs_refcount_btrec_to_irec(rec, irec); - agno = cur->bc_private.a.agno; + agno = cur->bc_ag.agno; if (irec->rc_blockcount == 0 || irec->rc_blockcount > MAXREFCEXTLEN) goto out_bad_rec; @@ -144,7 +144,7 @@ xfs_refcount_get_rec( if (irec->rc_refcount == 0 || irec->rc_refcount > MAXREFCOUNT) goto out_bad_rec; - trace_xfs_refcount_get(cur->bc_mp, cur->bc_private.a.agno, irec); + trace_xfs_refcount_get(cur->bc_mp, cur->bc_ag.agno, irec); return 0; out_bad_rec: @@ -169,14 +169,14 @@ xfs_refcount_update( union xfs_btree_rec rec; int error; - trace_xfs_refcount_update(cur->bc_mp, cur->bc_private.a.agno, irec); + trace_xfs_refcount_update(cur->bc_mp, cur->bc_ag.agno, irec); rec.refc.rc_startblock = cpu_to_be32(irec->rc_startblock); rec.refc.rc_blockcount = cpu_to_be32(irec->rc_blockcount); rec.refc.rc_refcount = cpu_to_be32(irec->rc_refcount); error = xfs_btree_update(cur, &rec); if (error) trace_xfs_refcount_update_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -193,7 +193,7 @@ xfs_refcount_insert( { int error; - trace_xfs_refcount_insert(cur->bc_mp, cur->bc_private.a.agno, irec); + trace_xfs_refcount_insert(cur->bc_mp, cur->bc_ag.agno, irec); cur->bc_rec.rc.rc_startblock = irec->rc_startblock; cur->bc_rec.rc.rc_blockcount = irec->rc_blockcount; cur->bc_rec.rc.rc_refcount = irec->rc_refcount; @@ -208,7 +208,7 @@ xfs_refcount_insert( out_error: if (error) trace_xfs_refcount_insert_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -234,7 +234,7 @@ xfs_refcount_delete( error = -EFSCORRUPTED; goto out_error; } - trace_xfs_refcount_delete(cur->bc_mp, cur->bc_private.a.agno, &irec); + trace_xfs_refcount_delete(cur->bc_mp, cur->bc_ag.agno, &irec); error = xfs_btree_delete(cur, i); if (XFS_IS_CORRUPT(cur->bc_mp, *i != 1)) { error = -EFSCORRUPTED; @@ -246,7 +246,7 @@ xfs_refcount_delete( out_error: if (error) trace_xfs_refcount_delete_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -366,7 +366,7 @@ xfs_refcount_split_extent( return 0; *shape_changed = true; - trace_xfs_refcount_split_extent(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcount_split_extent(cur->bc_mp, cur->bc_ag.agno, &rcext, agbno); /* Establish the right extent. */ @@ -391,7 +391,7 @@ xfs_refcount_split_extent( out_error: trace_xfs_refcount_split_extent_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -411,7 +411,7 @@ xfs_refcount_merge_center_extents( int found_rec; trace_xfs_refcount_merge_center_extents(cur->bc_mp, - cur->bc_private.a.agno, left, center, right); + cur->bc_ag.agno, left, center, right); /* * Make sure the center and right extents are not in the btree. @@ -468,7 +468,7 @@ xfs_refcount_merge_center_extents( out_error: trace_xfs_refcount_merge_center_extents_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -487,7 +487,7 @@ xfs_refcount_merge_left_extent( int found_rec; trace_xfs_refcount_merge_left_extent(cur->bc_mp, - cur->bc_private.a.agno, left, cleft); + cur->bc_ag.agno, left, cleft); /* If the extent at agbno (cleft) wasn't synthesized, remove it. */ if (cleft->rc_refcount > 1) { @@ -530,7 +530,7 @@ xfs_refcount_merge_left_extent( out_error: trace_xfs_refcount_merge_left_extent_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -548,7 +548,7 @@ xfs_refcount_merge_right_extent( int found_rec; trace_xfs_refcount_merge_right_extent(cur->bc_mp, - cur->bc_private.a.agno, cright, right); + cur->bc_ag.agno, cright, right); /* * If the extent ending at agbno+aglen (cright) wasn't synthesized, @@ -594,7 +594,7 @@ xfs_refcount_merge_right_extent( out_error: trace_xfs_refcount_merge_right_extent_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -679,13 +679,13 @@ xfs_refcount_find_left_extents( cleft->rc_blockcount = aglen; cleft->rc_refcount = 1; } - trace_xfs_refcount_find_left_extent(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcount_find_left_extent(cur->bc_mp, cur->bc_ag.agno, left, cleft, agbno); return error; out_error: trace_xfs_refcount_find_left_extent_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -768,13 +768,13 @@ xfs_refcount_find_right_extents( cright->rc_blockcount = aglen; cright->rc_refcount = 1; } - trace_xfs_refcount_find_right_extent(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcount_find_right_extent(cur->bc_mp, cur->bc_ag.agno, cright, right, agbno + aglen); return error; out_error: trace_xfs_refcount_find_right_extent_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -883,7 +883,7 @@ xfs_refcount_still_have_space( { unsigned long overhead; - overhead = cur->bc_private.a.priv.refc.shape_changes * + overhead = cur->bc_ag.refc.shape_changes * xfs_allocfree_log_count(cur->bc_mp, 1); overhead *= cur->bc_mp->m_sb.sb_blocksize; @@ -891,17 +891,17 @@ xfs_refcount_still_have_space( * Only allow 2 refcount extent updates per transaction if the * refcount continue update "error" has been injected. */ - if (cur->bc_private.a.priv.refc.nr_ops > 2 && + if (cur->bc_ag.refc.nr_ops > 2 && XFS_TEST_ERROR(false, cur->bc_mp, XFS_ERRTAG_REFCOUNT_CONTINUE_UPDATE)) return false; - if (cur->bc_private.a.priv.refc.nr_ops == 0) + if (cur->bc_ag.refc.nr_ops == 0) return true; else if (overhead > cur->bc_tp->t_log_res) return false; return cur->bc_tp->t_log_res - overhead > - cur->bc_private.a.priv.refc.nr_ops * XFS_REFCOUNT_ITEM_OVERHEAD; + cur->bc_ag.refc.nr_ops * XFS_REFCOUNT_ITEM_OVERHEAD; } /* @@ -952,7 +952,7 @@ xfs_refcount_adjust_extents( ext.rc_startblock - *agbno); tmp.rc_refcount = 1 + adj; trace_xfs_refcount_modify_extent(cur->bc_mp, - cur->bc_private.a.agno, &tmp); + cur->bc_ag.agno, &tmp); /* * Either cover the hole (increment) or @@ -968,10 +968,10 @@ xfs_refcount_adjust_extents( error = -EFSCORRUPTED; goto out_error; } - cur->bc_private.a.priv.refc.nr_ops++; + cur->bc_ag.refc.nr_ops++; } else { fsbno = XFS_AGB_TO_FSB(cur->bc_mp, - cur->bc_private.a.agno, + cur->bc_ag.agno, tmp.rc_startblock); xfs_bmap_add_free(cur->bc_tp, fsbno, tmp.rc_blockcount, oinfo); @@ -998,12 +998,12 @@ xfs_refcount_adjust_extents( goto skip; ext.rc_refcount += adj; trace_xfs_refcount_modify_extent(cur->bc_mp, - cur->bc_private.a.agno, &ext); + cur->bc_ag.agno, &ext); if (ext.rc_refcount > 1) { error = xfs_refcount_update(cur, &ext); if (error) goto out_error; - cur->bc_private.a.priv.refc.nr_ops++; + cur->bc_ag.refc.nr_ops++; } else if (ext.rc_refcount == 1) { error = xfs_refcount_delete(cur, &found_rec); if (error) @@ -1012,11 +1012,11 @@ xfs_refcount_adjust_extents( error = -EFSCORRUPTED; goto out_error; } - cur->bc_private.a.priv.refc.nr_ops++; + cur->bc_ag.refc.nr_ops++; goto advloop; } else { fsbno = XFS_AGB_TO_FSB(cur->bc_mp, - cur->bc_private.a.agno, + cur->bc_ag.agno, ext.rc_startblock); xfs_bmap_add_free(cur->bc_tp, fsbno, ext.rc_blockcount, oinfo); @@ -1035,7 +1035,7 @@ xfs_refcount_adjust_extents( return error; out_error: trace_xfs_refcount_modify_extent_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -1057,10 +1057,10 @@ xfs_refcount_adjust( *new_agbno = agbno; *new_aglen = aglen; if (adj == XFS_REFCOUNT_ADJUST_INCREASE) - trace_xfs_refcount_increase(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcount_increase(cur->bc_mp, cur->bc_ag.agno, agbno, aglen); else - trace_xfs_refcount_decrease(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcount_decrease(cur->bc_mp, cur->bc_ag.agno, agbno, aglen); /* @@ -1088,7 +1088,7 @@ xfs_refcount_adjust( if (shape_changed) shape_changes++; if (shape_changes) - cur->bc_private.a.priv.refc.shape_changes++; + cur->bc_ag.refc.shape_changes++; /* Now that we've taken care of the ends, adjust the middle extents */ error = xfs_refcount_adjust_extents(cur, new_agbno, new_aglen, @@ -1099,7 +1099,7 @@ xfs_refcount_adjust( return 0; out_error: - trace_xfs_refcount_adjust_error(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcount_adjust_error(cur->bc_mp, cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -1115,7 +1115,7 @@ xfs_refcount_finish_one_cleanup( if (rcur == NULL) return; - agbp = rcur->bc_private.a.agbp; + agbp = rcur->bc_ag.agbp; xfs_btree_del_cursor(rcur, error); if (error) xfs_trans_brelse(tp, agbp); @@ -1165,9 +1165,9 @@ xfs_refcount_finish_one( * the startblock, get one now. */ rcur = *pcur; - if (rcur != NULL && rcur->bc_private.a.agno != agno) { - nr_ops = rcur->bc_private.a.priv.refc.nr_ops; - shape_changes = rcur->bc_private.a.priv.refc.shape_changes; + if (rcur != NULL && rcur->bc_ag.agno != agno) { + nr_ops = rcur->bc_ag.refc.nr_ops; + shape_changes = rcur->bc_ag.refc.shape_changes; xfs_refcount_finish_one_cleanup(tp, rcur, 0); rcur = NULL; *pcur = NULL; @@ -1183,8 +1183,8 @@ xfs_refcount_finish_one( error = -ENOMEM; goto out_cur; } - rcur->bc_private.a.priv.refc.nr_ops = nr_ops; - rcur->bc_private.a.priv.refc.shape_changes = shape_changes; + rcur->bc_ag.refc.nr_ops = nr_ops; + rcur->bc_ag.refc.shape_changes = shape_changes; } *pcur = rcur; @@ -1303,7 +1303,7 @@ xfs_refcount_find_shared( int have; int error; - trace_xfs_refcount_find_shared(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcount_find_shared(cur->bc_mp, cur->bc_ag.agno, agbno, aglen); /* By default, skip the whole range */ @@ -1383,12 +1383,12 @@ xfs_refcount_find_shared( done: trace_xfs_refcount_find_shared_result(cur->bc_mp, - cur->bc_private.a.agno, *fbno, *flen); + cur->bc_ag.agno, *fbno, *flen); out_error: if (error) trace_xfs_refcount_find_shared_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -1485,7 +1485,7 @@ xfs_refcount_adjust_cow_extents( tmp.rc_blockcount = aglen; tmp.rc_refcount = 1; trace_xfs_refcount_modify_extent(cur->bc_mp, - cur->bc_private.a.agno, &tmp); + cur->bc_ag.agno, &tmp); error = xfs_refcount_insert(cur, &tmp, &found_tmp); @@ -1513,7 +1513,7 @@ xfs_refcount_adjust_cow_extents( ext.rc_refcount = 0; trace_xfs_refcount_modify_extent(cur->bc_mp, - cur->bc_private.a.agno, &ext); + cur->bc_ag.agno, &ext); error = xfs_refcount_delete(cur, &found_rec); if (error) goto out_error; @@ -1529,7 +1529,7 @@ xfs_refcount_adjust_cow_extents( return error; out_error: trace_xfs_refcount_modify_extent_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -1575,7 +1575,7 @@ xfs_refcount_adjust_cow( return 0; out_error: - trace_xfs_refcount_adjust_cow_error(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcount_adjust_cow_error(cur->bc_mp, cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -1589,7 +1589,7 @@ __xfs_refcount_cow_alloc( xfs_agblock_t agbno, xfs_extlen_t aglen) { - trace_xfs_refcount_cow_increase(rcur->bc_mp, rcur->bc_private.a.agno, + trace_xfs_refcount_cow_increase(rcur->bc_mp, rcur->bc_ag.agno, agbno, aglen); /* Add refcount btree reservation */ @@ -1606,7 +1606,7 @@ __xfs_refcount_cow_free( xfs_agblock_t agbno, xfs_extlen_t aglen) { - trace_xfs_refcount_cow_decrease(rcur->bc_mp, rcur->bc_private.a.agno, + trace_xfs_refcount_cow_decrease(rcur->bc_mp, rcur->bc_ag.agno, agbno, aglen); /* Remove refcount btree reservation */ diff --git a/fs/xfs/libxfs/xfs_refcount_btree.c b/fs/xfs/libxfs/xfs_refcount_btree.c index 38529dbacd55..7fd6044a4f78 100644 --- a/fs/xfs/libxfs/xfs_refcount_btree.c +++ b/fs/xfs/libxfs/xfs_refcount_btree.c @@ -12,6 +12,7 @@ #include "xfs_sb.h" #include "xfs_mount.h" #include "xfs_btree.h" +#include "xfs_btree_staging.h" #include "xfs_refcount_btree.h" #include "xfs_alloc.h" #include "xfs_error.h" @@ -25,7 +26,7 @@ xfs_refcountbt_dup_cursor( struct xfs_btree_cur *cur) { return xfs_refcountbt_init_cursor(cur->bc_mp, cur->bc_tp, - cur->bc_private.a.agbp, cur->bc_private.a.agno); + cur->bc_ag.agbp, cur->bc_ag.agno); } STATIC void @@ -34,8 +35,8 @@ xfs_refcountbt_set_root( union xfs_btree_ptr *ptr, int inc) { - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; xfs_agnumber_t seqno = be32_to_cpu(agf->agf_seqno); struct xfs_perag *pag = xfs_perag_get(cur->bc_mp, seqno); @@ -57,8 +58,8 @@ xfs_refcountbt_alloc_block( union xfs_btree_ptr *new, int *stat) { - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; struct xfs_alloc_arg args; /* block allocation args */ int error; /* error return value */ @@ -66,7 +67,7 @@ xfs_refcountbt_alloc_block( args.tp = cur->bc_tp; args.mp = cur->bc_mp; args.type = XFS_ALLOCTYPE_NEAR_BNO; - args.fsbno = XFS_AGB_TO_FSB(cur->bc_mp, cur->bc_private.a.agno, + args.fsbno = XFS_AGB_TO_FSB(cur->bc_mp, cur->bc_ag.agno, xfs_refc_block(args.mp)); args.oinfo = XFS_RMAP_OINFO_REFC; args.minlen = args.maxlen = args.prod = 1; @@ -75,13 +76,13 @@ xfs_refcountbt_alloc_block( error = xfs_alloc_vextent(&args); if (error) goto out_error; - trace_xfs_refcountbt_alloc_block(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcountbt_alloc_block(cur->bc_mp, cur->bc_ag.agno, args.agbno, 1); if (args.fsbno == NULLFSBLOCK) { *stat = 0; return 0; } - ASSERT(args.agno == cur->bc_private.a.agno); + ASSERT(args.agno == cur->bc_ag.agno); ASSERT(args.len == 1); new->s = cpu_to_be32(args.agbno); @@ -101,12 +102,12 @@ xfs_refcountbt_free_block( struct xfs_buf *bp) { struct xfs_mount *mp = cur->bc_mp; - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; xfs_fsblock_t fsbno = XFS_DADDR_TO_FSB(mp, XFS_BUF_ADDR(bp)); int error; - trace_xfs_refcountbt_free_block(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_refcountbt_free_block(cur->bc_mp, cur->bc_ag.agno, XFS_FSB_TO_AGBNO(cur->bc_mp, fsbno), 1); be32_add_cpu(&agf->agf_refcount_blocks, -1); xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_REFCOUNT_BLOCKS); @@ -169,9 +170,9 @@ xfs_refcountbt_init_ptr_from_cur( struct xfs_btree_cur *cur, union xfs_btree_ptr *ptr) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(cur->bc_private.a.agbp); + struct xfs_agf *agf = cur->bc_ag.agbp->b_addr; - ASSERT(cur->bc_private.a.agno == be32_to_cpu(agf->agf_seqno)); + ASSERT(cur->bc_ag.agno == be32_to_cpu(agf->agf_seqno)); ptr->s = agf->agf_refcount_root; } @@ -311,8 +312,36 @@ static const struct xfs_btree_ops xfs_refcountbt_ops = { }; /* - * Allocate a new refcount btree cursor. + * Initialize a new refcount btree cursor. */ +static struct xfs_btree_cur * +xfs_refcountbt_init_common( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_agnumber_t agno) +{ + struct xfs_btree_cur *cur; + + ASSERT(agno != NULLAGNUMBER); + ASSERT(agno < mp->m_sb.sb_agcount); + + cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS); + cur->bc_tp = tp; + cur->bc_mp = mp; + cur->bc_btnum = XFS_BTNUM_REFC; + cur->bc_blocklog = mp->m_sb.sb_blocklog; + cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_refcbt_2); + + cur->bc_ag.agno = agno; + cur->bc_flags |= XFS_BTREE_CRC_BLOCKS; + + cur->bc_ag.refc.nr_ops = 0; + cur->bc_ag.refc.shape_changes = 0; + cur->bc_ops = &xfs_refcountbt_ops; + return cur; +} + +/* Create a btree cursor. */ struct xfs_btree_cur * xfs_refcountbt_init_cursor( struct xfs_mount *mp, @@ -320,32 +349,53 @@ xfs_refcountbt_init_cursor( struct xfs_buf *agbp, xfs_agnumber_t agno) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_agf *agf = agbp->b_addr; struct xfs_btree_cur *cur; - ASSERT(agno != NULLAGNUMBER); - ASSERT(agno < mp->m_sb.sb_agcount); - cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS); - - cur->bc_tp = tp; - cur->bc_mp = mp; - cur->bc_btnum = XFS_BTNUM_REFC; - cur->bc_blocklog = mp->m_sb.sb_blocklog; - cur->bc_ops = &xfs_refcountbt_ops; - cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_refcbt_2); - + cur = xfs_refcountbt_init_common(mp, tp, agno); cur->bc_nlevels = be32_to_cpu(agf->agf_refcount_level); - - cur->bc_private.a.agbp = agbp; - cur->bc_private.a.agno = agno; - cur->bc_flags |= XFS_BTREE_CRC_BLOCKS; - - cur->bc_private.a.priv.refc.nr_ops = 0; - cur->bc_private.a.priv.refc.shape_changes = 0; - + cur->bc_ag.agbp = agbp; return cur; } +/* Create a btree cursor with a fake root for staging. */ +struct xfs_btree_cur * +xfs_refcountbt_stage_cursor( + struct xfs_mount *mp, + struct xbtree_afakeroot *afake, + xfs_agnumber_t agno) +{ + struct xfs_btree_cur *cur; + + cur = xfs_refcountbt_init_common(mp, NULL, agno); + xfs_btree_stage_afakeroot(cur, afake); + return cur; +} + +/* + * Swap in the new btree root. Once we pass this point the newly rebuilt btree + * is in place and we have to kill off all the old btree blocks. + */ +void +xfs_refcountbt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_agf *agf = agbp->b_addr; + struct xbtree_afakeroot *afake = cur->bc_ag.afake; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + agf->agf_refcount_root = cpu_to_be32(afake->af_root); + agf->agf_refcount_level = cpu_to_be32(afake->af_levels); + agf->agf_refcount_blocks = cpu_to_be32(afake->af_blocks); + xfs_alloc_log_agf(tp, agbp, XFS_AGF_REFCOUNT_BLOCKS | + XFS_AGF_REFCOUNT_ROOT | + XFS_AGF_REFCOUNT_LEVEL); + xfs_btree_commit_afakeroot(cur, tp, agbp, &xfs_refcountbt_ops); +} + /* * Calculate the number of records in a refcount btree block. */ @@ -420,7 +470,7 @@ xfs_refcountbt_calc_reserves( if (error) return error; - agf = XFS_BUF_TO_AGF(agbp); + agf = agbp->b_addr; agblocks = be32_to_cpu(agf->agf_length); tree_len = be32_to_cpu(agf->agf_refcount_blocks); xfs_trans_brelse(tp, agbp); diff --git a/fs/xfs/libxfs/xfs_refcount_btree.h b/fs/xfs/libxfs/xfs_refcount_btree.h index ba416f71c824..69dc515db671 100644 --- a/fs/xfs/libxfs/xfs_refcount_btree.h +++ b/fs/xfs/libxfs/xfs_refcount_btree.h @@ -13,6 +13,7 @@ struct xfs_buf; struct xfs_btree_cur; struct xfs_mount; +struct xbtree_afakeroot; /* * Btree block header size @@ -46,6 +47,8 @@ struct xfs_mount; extern struct xfs_btree_cur *xfs_refcountbt_init_cursor(struct xfs_mount *mp, struct xfs_trans *tp, struct xfs_buf *agbp, xfs_agnumber_t agno); +struct xfs_btree_cur *xfs_refcountbt_stage_cursor(struct xfs_mount *mp, + struct xbtree_afakeroot *afake, xfs_agnumber_t agno); extern int xfs_refcountbt_maxrecs(int blocklen, bool leaf); extern void xfs_refcountbt_compute_maxlevels(struct xfs_mount *mp); @@ -58,4 +61,7 @@ extern int xfs_refcountbt_calc_reserves(struct xfs_mount *mp, struct xfs_trans *tp, xfs_agnumber_t agno, xfs_extlen_t *ask, xfs_extlen_t *used); +void xfs_refcountbt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp, struct xfs_buf *agbp); + #endif /* __XFS_REFCOUNT_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_rmap.c b/fs/xfs/libxfs/xfs_rmap.c index ff9412f113c4..27c39268c31f 100644 --- a/fs/xfs/libxfs/xfs_rmap.c +++ b/fs/xfs/libxfs/xfs_rmap.c @@ -79,7 +79,7 @@ xfs_rmap_update( union xfs_btree_rec rec; int error; - trace_xfs_rmap_update(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_rmap_update(cur->bc_mp, cur->bc_ag.agno, irec->rm_startblock, irec->rm_blockcount, irec->rm_owner, irec->rm_offset, irec->rm_flags); @@ -91,7 +91,7 @@ xfs_rmap_update( error = xfs_btree_update(cur, &rec); if (error) trace_xfs_rmap_update_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -107,7 +107,7 @@ xfs_rmap_insert( int i; int error; - trace_xfs_rmap_insert(rcur->bc_mp, rcur->bc_private.a.agno, agbno, + trace_xfs_rmap_insert(rcur->bc_mp, rcur->bc_ag.agno, agbno, len, owner, offset, flags); error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i); @@ -133,7 +133,7 @@ xfs_rmap_insert( done: if (error) trace_xfs_rmap_insert_error(rcur->bc_mp, - rcur->bc_private.a.agno, error, _RET_IP_); + rcur->bc_ag.agno, error, _RET_IP_); return error; } @@ -149,7 +149,7 @@ xfs_rmap_delete( int i; int error; - trace_xfs_rmap_delete(rcur->bc_mp, rcur->bc_private.a.agno, agbno, + trace_xfs_rmap_delete(rcur->bc_mp, rcur->bc_ag.agno, agbno, len, owner, offset, flags); error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i); @@ -170,7 +170,7 @@ xfs_rmap_delete( done: if (error) trace_xfs_rmap_delete_error(rcur->bc_mp, - rcur->bc_private.a.agno, error, _RET_IP_); + rcur->bc_ag.agno, error, _RET_IP_); return error; } @@ -197,7 +197,7 @@ xfs_rmap_get_rec( int *stat) { struct xfs_mount *mp = cur->bc_mp; - xfs_agnumber_t agno = cur->bc_private.a.agno; + xfs_agnumber_t agno = cur->bc_ag.agno; union xfs_btree_rec *rec; int error; @@ -260,7 +260,7 @@ xfs_rmap_find_left_neighbor_helper( struct xfs_find_left_neighbor_info *info = priv; trace_xfs_rmap_find_left_neighbor_candidate(cur->bc_mp, - cur->bc_private.a.agno, rec->rm_startblock, + cur->bc_ag.agno, rec->rm_startblock, rec->rm_blockcount, rec->rm_owner, rec->rm_offset, rec->rm_flags); @@ -312,7 +312,7 @@ xfs_rmap_find_left_neighbor( info.stat = stat; trace_xfs_rmap_find_left_neighbor_query(cur->bc_mp, - cur->bc_private.a.agno, bno, 0, owner, offset, flags); + cur->bc_ag.agno, bno, 0, owner, offset, flags); error = xfs_rmap_query_range(cur, &info.high, &info.high, xfs_rmap_find_left_neighbor_helper, &info); @@ -320,7 +320,7 @@ xfs_rmap_find_left_neighbor( error = 0; if (*stat) trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp, - cur->bc_private.a.agno, irec->rm_startblock, + cur->bc_ag.agno, irec->rm_startblock, irec->rm_blockcount, irec->rm_owner, irec->rm_offset, irec->rm_flags); return error; @@ -336,7 +336,7 @@ xfs_rmap_lookup_le_range_helper( struct xfs_find_left_neighbor_info *info = priv; trace_xfs_rmap_lookup_le_range_candidate(cur->bc_mp, - cur->bc_private.a.agno, rec->rm_startblock, + cur->bc_ag.agno, rec->rm_startblock, rec->rm_blockcount, rec->rm_owner, rec->rm_offset, rec->rm_flags); @@ -385,14 +385,14 @@ xfs_rmap_lookup_le_range( info.stat = stat; trace_xfs_rmap_lookup_le_range(cur->bc_mp, - cur->bc_private.a.agno, bno, 0, owner, offset, flags); + cur->bc_ag.agno, bno, 0, owner, offset, flags); error = xfs_rmap_query_range(cur, &info.high, &info.high, xfs_rmap_lookup_le_range_helper, &info); if (error == -ECANCELED) error = 0; if (*stat) trace_xfs_rmap_lookup_le_range_result(cur->bc_mp, - cur->bc_private.a.agno, irec->rm_startblock, + cur->bc_ag.agno, irec->rm_startblock, irec->rm_blockcount, irec->rm_owner, irec->rm_offset, irec->rm_flags); return error; @@ -498,7 +498,7 @@ xfs_rmap_unmap( (flags & XFS_RMAP_BMBT_BLOCK); if (unwritten) flags |= XFS_RMAP_UNWRITTEN; - trace_xfs_rmap_unmap(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_unmap(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); /* @@ -522,7 +522,7 @@ xfs_rmap_unmap( goto out_error; } trace_xfs_rmap_lookup_le_range_result(cur->bc_mp, - cur->bc_private.a.agno, ltrec.rm_startblock, + cur->bc_ag.agno, ltrec.rm_startblock, ltrec.rm_blockcount, ltrec.rm_owner, ltrec.rm_offset, ltrec.rm_flags); ltoff = ltrec.rm_offset; @@ -588,7 +588,7 @@ xfs_rmap_unmap( if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) { /* exact match, simply remove the record from rmap tree */ - trace_xfs_rmap_delete(mp, cur->bc_private.a.agno, + trace_xfs_rmap_delete(mp, cur->bc_ag.agno, ltrec.rm_startblock, ltrec.rm_blockcount, ltrec.rm_owner, ltrec.rm_offset, ltrec.rm_flags); @@ -666,7 +666,7 @@ xfs_rmap_unmap( else cur->bc_rec.r.rm_offset = offset + len; cur->bc_rec.r.rm_flags = flags; - trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, + trace_xfs_rmap_insert(mp, cur->bc_ag.agno, cur->bc_rec.r.rm_startblock, cur->bc_rec.r.rm_blockcount, cur->bc_rec.r.rm_owner, @@ -678,11 +678,11 @@ xfs_rmap_unmap( } out_done: - trace_xfs_rmap_unmap_done(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_unmap_done(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); out_error: if (error) - trace_xfs_rmap_unmap_error(mp, cur->bc_private.a.agno, + trace_xfs_rmap_unmap_error(mp, cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -773,7 +773,7 @@ xfs_rmap_map( (flags & XFS_RMAP_BMBT_BLOCK); if (unwritten) flags |= XFS_RMAP_UNWRITTEN; - trace_xfs_rmap_map(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_map(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); ASSERT(!xfs_rmap_should_skip_owner_update(oinfo)); @@ -795,7 +795,7 @@ xfs_rmap_map( goto out_error; } trace_xfs_rmap_lookup_le_range_result(cur->bc_mp, - cur->bc_private.a.agno, ltrec.rm_startblock, + cur->bc_ag.agno, ltrec.rm_startblock, ltrec.rm_blockcount, ltrec.rm_owner, ltrec.rm_offset, ltrec.rm_flags); @@ -831,7 +831,7 @@ xfs_rmap_map( goto out_error; } trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp, - cur->bc_private.a.agno, gtrec.rm_startblock, + cur->bc_ag.agno, gtrec.rm_startblock, gtrec.rm_blockcount, gtrec.rm_owner, gtrec.rm_offset, gtrec.rm_flags); if (!xfs_rmap_is_mergeable(>rec, owner, flags)) @@ -870,7 +870,7 @@ xfs_rmap_map( * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr| */ ltrec.rm_blockcount += gtrec.rm_blockcount; - trace_xfs_rmap_delete(mp, cur->bc_private.a.agno, + trace_xfs_rmap_delete(mp, cur->bc_ag.agno, gtrec.rm_startblock, gtrec.rm_blockcount, gtrec.rm_owner, @@ -921,7 +921,7 @@ xfs_rmap_map( cur->bc_rec.r.rm_owner = owner; cur->bc_rec.r.rm_offset = offset; cur->bc_rec.r.rm_flags = flags; - trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_insert(mp, cur->bc_ag.agno, bno, len, owner, offset, flags); error = xfs_btree_insert(cur, &i); if (error) @@ -932,11 +932,11 @@ xfs_rmap_map( } } - trace_xfs_rmap_map_done(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_map_done(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); out_error: if (error) - trace_xfs_rmap_map_error(mp, cur->bc_private.a.agno, + trace_xfs_rmap_map_error(mp, cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -1010,7 +1010,7 @@ xfs_rmap_convert( (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK)))); oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0; new_endoff = offset + len; - trace_xfs_rmap_convert(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_convert(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); /* @@ -1034,7 +1034,7 @@ xfs_rmap_convert( goto done; } trace_xfs_rmap_lookup_le_range_result(cur->bc_mp, - cur->bc_private.a.agno, PREV.rm_startblock, + cur->bc_ag.agno, PREV.rm_startblock, PREV.rm_blockcount, PREV.rm_owner, PREV.rm_offset, PREV.rm_flags); @@ -1076,7 +1076,7 @@ xfs_rmap_convert( goto done; } trace_xfs_rmap_find_left_neighbor_result(cur->bc_mp, - cur->bc_private.a.agno, LEFT.rm_startblock, + cur->bc_ag.agno, LEFT.rm_startblock, LEFT.rm_blockcount, LEFT.rm_owner, LEFT.rm_offset, LEFT.rm_flags); if (LEFT.rm_startblock + LEFT.rm_blockcount == bno && @@ -1114,7 +1114,7 @@ xfs_rmap_convert( goto done; } trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp, - cur->bc_private.a.agno, RIGHT.rm_startblock, + cur->bc_ag.agno, RIGHT.rm_startblock, RIGHT.rm_blockcount, RIGHT.rm_owner, RIGHT.rm_offset, RIGHT.rm_flags); if (bno + len == RIGHT.rm_startblock && @@ -1132,7 +1132,7 @@ xfs_rmap_convert( RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX) state &= ~RMAP_RIGHT_CONTIG; - trace_xfs_rmap_convert_state(mp, cur->bc_private.a.agno, state, + trace_xfs_rmap_convert_state(mp, cur->bc_ag.agno, state, _RET_IP_); /* reset the cursor back to PREV */ @@ -1162,7 +1162,7 @@ xfs_rmap_convert( error = -EFSCORRUPTED; goto done; } - trace_xfs_rmap_delete(mp, cur->bc_private.a.agno, + trace_xfs_rmap_delete(mp, cur->bc_ag.agno, RIGHT.rm_startblock, RIGHT.rm_blockcount, RIGHT.rm_owner, RIGHT.rm_offset, RIGHT.rm_flags); @@ -1180,7 +1180,7 @@ xfs_rmap_convert( error = -EFSCORRUPTED; goto done; } - trace_xfs_rmap_delete(mp, cur->bc_private.a.agno, + trace_xfs_rmap_delete(mp, cur->bc_ag.agno, PREV.rm_startblock, PREV.rm_blockcount, PREV.rm_owner, PREV.rm_offset, PREV.rm_flags); @@ -1210,7 +1210,7 @@ xfs_rmap_convert( * Setting all of a previous oldext extent to newext. * The left neighbor is contiguous, the right is not. */ - trace_xfs_rmap_delete(mp, cur->bc_private.a.agno, + trace_xfs_rmap_delete(mp, cur->bc_ag.agno, PREV.rm_startblock, PREV.rm_blockcount, PREV.rm_owner, PREV.rm_offset, PREV.rm_flags); @@ -1247,7 +1247,7 @@ xfs_rmap_convert( error = -EFSCORRUPTED; goto done; } - trace_xfs_rmap_delete(mp, cur->bc_private.a.agno, + trace_xfs_rmap_delete(mp, cur->bc_ag.agno, RIGHT.rm_startblock, RIGHT.rm_blockcount, RIGHT.rm_owner, RIGHT.rm_offset, RIGHT.rm_flags); @@ -1326,7 +1326,7 @@ xfs_rmap_convert( NEW.rm_blockcount = len; NEW.rm_flags = newext; cur->bc_rec.r = NEW; - trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, bno, + trace_xfs_rmap_insert(mp, cur->bc_ag.agno, bno, len, owner, offset, newext); error = xfs_btree_insert(cur, &i); if (error) @@ -1383,7 +1383,7 @@ xfs_rmap_convert( NEW.rm_blockcount = len; NEW.rm_flags = newext; cur->bc_rec.r = NEW; - trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, bno, + trace_xfs_rmap_insert(mp, cur->bc_ag.agno, bno, len, owner, offset, newext); error = xfs_btree_insert(cur, &i); if (error) @@ -1414,7 +1414,7 @@ xfs_rmap_convert( NEW = PREV; NEW.rm_blockcount = offset - PREV.rm_offset; cur->bc_rec.r = NEW; - trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, + trace_xfs_rmap_insert(mp, cur->bc_ag.agno, NEW.rm_startblock, NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset, NEW.rm_flags); @@ -1441,7 +1441,7 @@ xfs_rmap_convert( /* new middle extent - newext */ cur->bc_rec.r.rm_flags &= ~XFS_RMAP_UNWRITTEN; cur->bc_rec.r.rm_flags |= newext; - trace_xfs_rmap_insert(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_insert(mp, cur->bc_ag.agno, bno, len, owner, offset, newext); error = xfs_btree_insert(cur, &i); if (error) @@ -1465,12 +1465,12 @@ xfs_rmap_convert( ASSERT(0); } - trace_xfs_rmap_convert_done(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_convert_done(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); done: if (error) trace_xfs_rmap_convert_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -1506,7 +1506,7 @@ xfs_rmap_convert_shared( (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK)))); oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0; new_endoff = offset + len; - trace_xfs_rmap_convert(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_convert(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); /* @@ -1573,7 +1573,7 @@ xfs_rmap_convert_shared( goto done; } trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp, - cur->bc_private.a.agno, RIGHT.rm_startblock, + cur->bc_ag.agno, RIGHT.rm_startblock, RIGHT.rm_blockcount, RIGHT.rm_owner, RIGHT.rm_offset, RIGHT.rm_flags); if (xfs_rmap_is_mergeable(&RIGHT, owner, newext)) @@ -1589,7 +1589,7 @@ xfs_rmap_convert_shared( RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX) state &= ~RMAP_RIGHT_CONTIG; - trace_xfs_rmap_convert_state(mp, cur->bc_private.a.agno, state, + trace_xfs_rmap_convert_state(mp, cur->bc_ag.agno, state, _RET_IP_); /* * Switch out based on the FILLING and CONTIG state bits. @@ -1880,12 +1880,12 @@ xfs_rmap_convert_shared( ASSERT(0); } - trace_xfs_rmap_convert_done(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_convert_done(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); done: if (error) trace_xfs_rmap_convert_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -1923,7 +1923,7 @@ xfs_rmap_unmap_shared( xfs_owner_info_unpack(oinfo, &owner, &offset, &flags); if (unwritten) flags |= XFS_RMAP_UNWRITTEN; - trace_xfs_rmap_unmap(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_unmap(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); /* @@ -2072,12 +2072,12 @@ xfs_rmap_unmap_shared( goto out_error; } - trace_xfs_rmap_unmap_done(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_unmap_done(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); out_error: if (error) trace_xfs_rmap_unmap_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -2112,7 +2112,7 @@ xfs_rmap_map_shared( xfs_owner_info_unpack(oinfo, &owner, &offset, &flags); if (unwritten) flags |= XFS_RMAP_UNWRITTEN; - trace_xfs_rmap_map(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_map(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); /* Is there a left record that abuts our range? */ @@ -2138,7 +2138,7 @@ xfs_rmap_map_shared( goto out_error; } trace_xfs_rmap_find_right_neighbor_result(cur->bc_mp, - cur->bc_private.a.agno, gtrec.rm_startblock, + cur->bc_ag.agno, gtrec.rm_startblock, gtrec.rm_blockcount, gtrec.rm_owner, gtrec.rm_offset, gtrec.rm_flags); @@ -2231,12 +2231,12 @@ xfs_rmap_map_shared( goto out_error; } - trace_xfs_rmap_map_done(mp, cur->bc_private.a.agno, bno, len, + trace_xfs_rmap_map_done(mp, cur->bc_ag.agno, bno, len, unwritten, oinfo); out_error: if (error) trace_xfs_rmap_map_error(cur->bc_mp, - cur->bc_private.a.agno, error, _RET_IP_); + cur->bc_ag.agno, error, _RET_IP_); return error; } @@ -2336,7 +2336,7 @@ xfs_rmap_finish_one_cleanup( if (rcur == NULL) return; - agbp = rcur->bc_private.a.agbp; + agbp = rcur->bc_ag.agbp; xfs_btree_del_cursor(rcur, error); if (error) xfs_trans_brelse(tp, agbp); @@ -2386,7 +2386,7 @@ xfs_rmap_finish_one( * the startblock, get one now. */ rcur = *pcur; - if (rcur != NULL && rcur->bc_private.a.agno != agno) { + if (rcur != NULL && rcur->bc_ag.agno != agno) { xfs_rmap_finish_one_cleanup(tp, rcur, 0); rcur = NULL; *pcur = NULL; @@ -2694,7 +2694,6 @@ struct xfs_rmap_key_state { uint64_t owner; uint64_t offset; unsigned int flags; - bool has_rmap; }; /* For each rmap given, figure out if it doesn't match the key we want. */ @@ -2709,7 +2708,6 @@ xfs_rmap_has_other_keys_helper( if (rks->owner == rec->rm_owner && rks->offset == rec->rm_offset && ((rks->flags & rec->rm_flags) & XFS_RMAP_KEY_FLAGS) == rks->flags) return 0; - rks->has_rmap = true; return -ECANCELED; } @@ -2731,7 +2729,7 @@ xfs_rmap_has_other_keys( int error; xfs_owner_info_unpack(oinfo, &rks.owner, &rks.offset, &rks.flags); - rks.has_rmap = false; + *has_rmap = false; low.rm_startblock = bno; memset(&high, 0xFF, sizeof(high)); @@ -2739,11 +2737,12 @@ xfs_rmap_has_other_keys( error = xfs_rmap_query_range(cur, &low, &high, xfs_rmap_has_other_keys_helper, &rks); - if (error < 0) - return error; + if (error == -ECANCELED) { + *has_rmap = true; + return 0; + } - *has_rmap = rks.has_rmap; - return 0; + return error; } const struct xfs_owner_info XFS_RMAP_OINFO_SKIP_UPDATE = { diff --git a/fs/xfs/libxfs/xfs_rmap_btree.c b/fs/xfs/libxfs/xfs_rmap_btree.c index fc78efa52c94..b7c05314d07c 100644 --- a/fs/xfs/libxfs/xfs_rmap_btree.c +++ b/fs/xfs/libxfs/xfs_rmap_btree.c @@ -14,6 +14,7 @@ #include "xfs_trans.h" #include "xfs_alloc.h" #include "xfs_btree.h" +#include "xfs_btree_staging.h" #include "xfs_rmap.h" #include "xfs_rmap_btree.h" #include "xfs_trace.h" @@ -51,7 +52,7 @@ xfs_rmapbt_dup_cursor( struct xfs_btree_cur *cur) { return xfs_rmapbt_init_cursor(cur->bc_mp, cur->bc_tp, - cur->bc_private.a.agbp, cur->bc_private.a.agno); + cur->bc_ag.agbp, cur->bc_ag.agno); } STATIC void @@ -60,8 +61,8 @@ xfs_rmapbt_set_root( union xfs_btree_ptr *ptr, int inc) { - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; xfs_agnumber_t seqno = be32_to_cpu(agf->agf_seqno); int btnum = cur->bc_btnum; struct xfs_perag *pag = xfs_perag_get(cur->bc_mp, seqno); @@ -83,25 +84,25 @@ xfs_rmapbt_alloc_block( union xfs_btree_ptr *new, int *stat) { - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; int error; xfs_agblock_t bno; /* Allocate the new block from the freelist. If we can't, give up. */ - error = xfs_alloc_get_freelist(cur->bc_tp, cur->bc_private.a.agbp, + error = xfs_alloc_get_freelist(cur->bc_tp, cur->bc_ag.agbp, &bno, 1); if (error) return error; - trace_xfs_rmapbt_alloc_block(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_rmapbt_alloc_block(cur->bc_mp, cur->bc_ag.agno, bno, 1); if (bno == NULLAGBLOCK) { *stat = 0; return 0; } - xfs_extent_busy_reuse(cur->bc_mp, cur->bc_private.a.agno, bno, 1, + xfs_extent_busy_reuse(cur->bc_mp, cur->bc_ag.agno, bno, 1, false); xfs_trans_agbtree_delta(cur->bc_tp, 1); @@ -109,7 +110,7 @@ xfs_rmapbt_alloc_block( be32_add_cpu(&agf->agf_rmap_blocks, 1); xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_RMAP_BLOCKS); - xfs_ag_resv_rmapbt_alloc(cur->bc_mp, cur->bc_private.a.agno); + xfs_ag_resv_rmapbt_alloc(cur->bc_mp, cur->bc_ag.agno); *stat = 1; return 0; @@ -120,13 +121,13 @@ xfs_rmapbt_free_block( struct xfs_btree_cur *cur, struct xfs_buf *bp) { - struct xfs_buf *agbp = cur->bc_private.a.agbp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; xfs_agblock_t bno; int error; bno = xfs_daddr_to_agbno(cur->bc_mp, XFS_BUF_ADDR(bp)); - trace_xfs_rmapbt_free_block(cur->bc_mp, cur->bc_private.a.agno, + trace_xfs_rmapbt_free_block(cur->bc_mp, cur->bc_ag.agno, bno, 1); be32_add_cpu(&agf->agf_rmap_blocks, -1); xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_RMAP_BLOCKS); @@ -138,7 +139,7 @@ xfs_rmapbt_free_block( XFS_EXTENT_BUSY_SKIP_DISCARD); xfs_trans_agbtree_delta(cur->bc_tp, -1); - xfs_ag_resv_rmapbt_free(cur->bc_mp, cur->bc_private.a.agno); + xfs_ag_resv_rmapbt_free(cur->bc_mp, cur->bc_ag.agno); return 0; } @@ -215,9 +216,9 @@ xfs_rmapbt_init_ptr_from_cur( struct xfs_btree_cur *cur, union xfs_btree_ptr *ptr) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(cur->bc_private.a.agbp); + struct xfs_agf *agf = cur->bc_ag.agbp->b_addr; - ASSERT(cur->bc_private.a.agno == be32_to_cpu(agf->agf_seqno)); + ASSERT(cur->bc_ag.agno == be32_to_cpu(agf->agf_seqno)); ptr->s = agf->agf_roots[cur->bc_btnum]; } @@ -448,17 +449,12 @@ static const struct xfs_btree_ops xfs_rmapbt_ops = { .recs_inorder = xfs_rmapbt_recs_inorder, }; -/* - * Allocate a new allocation btree cursor. - */ -struct xfs_btree_cur * -xfs_rmapbt_init_cursor( +static struct xfs_btree_cur * +xfs_rmapbt_init_common( struct xfs_mount *mp, struct xfs_trans *tp, - struct xfs_buf *agbp, xfs_agnumber_t agno) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(agbp); struct xfs_btree_cur *cur; cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS); @@ -468,16 +464,67 @@ xfs_rmapbt_init_cursor( cur->bc_btnum = XFS_BTNUM_RMAP; cur->bc_flags = XFS_BTREE_CRC_BLOCKS | XFS_BTREE_OVERLAPPING; cur->bc_blocklog = mp->m_sb.sb_blocklog; - cur->bc_ops = &xfs_rmapbt_ops; - cur->bc_nlevels = be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]); cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_rmap_2); - - cur->bc_private.a.agbp = agbp; - cur->bc_private.a.agno = agno; + cur->bc_ag.agno = agno; + cur->bc_ops = &xfs_rmapbt_ops; return cur; } +/* Create a new reverse mapping btree cursor. */ +struct xfs_btree_cur * +xfs_rmapbt_init_cursor( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_buf *agbp, + xfs_agnumber_t agno) +{ + struct xfs_agf *agf = agbp->b_addr; + struct xfs_btree_cur *cur; + + cur = xfs_rmapbt_init_common(mp, tp, agno); + cur->bc_nlevels = be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]); + cur->bc_ag.agbp = agbp; + return cur; +} + +/* Create a new reverse mapping btree cursor with a fake root for staging. */ +struct xfs_btree_cur * +xfs_rmapbt_stage_cursor( + struct xfs_mount *mp, + struct xbtree_afakeroot *afake, + xfs_agnumber_t agno) +{ + struct xfs_btree_cur *cur; + + cur = xfs_rmapbt_init_common(mp, NULL, agno); + xfs_btree_stage_afakeroot(cur, afake); + return cur; +} + +/* + * Install a new reverse mapping btree root. Caller is responsible for + * invalidating and freeing the old btree blocks. + */ +void +xfs_rmapbt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_agf *agf = agbp->b_addr; + struct xbtree_afakeroot *afake = cur->bc_ag.afake; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + agf->agf_roots[cur->bc_btnum] = cpu_to_be32(afake->af_root); + agf->agf_levels[cur->bc_btnum] = cpu_to_be32(afake->af_levels); + agf->agf_rmap_blocks = cpu_to_be32(afake->af_blocks); + xfs_alloc_log_agf(tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS | + XFS_AGF_RMAP_BLOCKS); + xfs_btree_commit_afakeroot(cur, tp, agbp, &xfs_rmapbt_ops); +} + /* * Calculate number of records in an rmap btree block. */ @@ -569,7 +616,7 @@ xfs_rmapbt_calc_reserves( if (error) return error; - agf = XFS_BUF_TO_AGF(agbp); + agf = agbp->b_addr; agblocks = be32_to_cpu(agf->agf_length); tree_len = be32_to_cpu(agf->agf_rmap_blocks); xfs_trans_brelse(tp, agbp); diff --git a/fs/xfs/libxfs/xfs_rmap_btree.h b/fs/xfs/libxfs/xfs_rmap_btree.h index 820d668b063d..115c3455a734 100644 --- a/fs/xfs/libxfs/xfs_rmap_btree.h +++ b/fs/xfs/libxfs/xfs_rmap_btree.h @@ -9,6 +9,7 @@ struct xfs_buf; struct xfs_btree_cur; struct xfs_mount; +struct xbtree_afakeroot; /* rmaps only exist on crc enabled filesystems */ #define XFS_RMAP_BLOCK_LEN XFS_BTREE_SBLOCK_CRC_LEN @@ -43,6 +44,10 @@ struct xfs_mount; struct xfs_btree_cur *xfs_rmapbt_init_cursor(struct xfs_mount *mp, struct xfs_trans *tp, struct xfs_buf *bp, xfs_agnumber_t agno); +struct xfs_btree_cur *xfs_rmapbt_stage_cursor(struct xfs_mount *mp, + struct xbtree_afakeroot *afake, xfs_agnumber_t agno); +void xfs_rmapbt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp, struct xfs_buf *agbp); int xfs_rmapbt_maxrecs(int blocklen, int leaf); extern void xfs_rmapbt_compute_maxlevels(struct xfs_mount *mp); diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c index 2f60fc3c99a0..00266de58954 100644 --- a/fs/xfs/libxfs/xfs_sb.c +++ b/fs/xfs/libxfs/xfs_sb.c @@ -220,7 +220,7 @@ xfs_validate_sb_common( struct xfs_buf *bp, struct xfs_sb *sbp) { - struct xfs_dsb *dsb = XFS_BUF_TO_SBP(bp); + struct xfs_dsb *dsb = bp->b_addr; uint32_t agcount = 0; uint32_t rem; @@ -681,7 +681,7 @@ xfs_sb_read_verify( { struct xfs_sb sb; struct xfs_mount *mp = bp->b_mount; - struct xfs_dsb *dsb = XFS_BUF_TO_SBP(bp); + struct xfs_dsb *dsb = bp->b_addr; int error; /* @@ -707,7 +707,7 @@ xfs_sb_read_verify( * Check all the superblock fields. Don't byteswap the xquota flags * because _verify_common checks the on-disk values. */ - __xfs_sb_from_disk(&sb, XFS_BUF_TO_SBP(bp), false); + __xfs_sb_from_disk(&sb, dsb, false); error = xfs_validate_sb_common(mp, bp, &sb); if (error) goto out_error; @@ -730,7 +730,7 @@ static void xfs_sb_quiet_read_verify( struct xfs_buf *bp) { - struct xfs_dsb *dsb = XFS_BUF_TO_SBP(bp); + struct xfs_dsb *dsb = bp->b_addr; if (dsb->sb_magicnum == cpu_to_be32(XFS_SB_MAGIC)) { /* XFS filesystem, verify noisily! */ @@ -748,13 +748,14 @@ xfs_sb_write_verify( struct xfs_sb sb; struct xfs_mount *mp = bp->b_mount; struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_dsb *dsb = bp->b_addr; int error; /* * Check all the superblock fields. Don't byteswap the xquota flags * because _verify_common checks the on-disk values. */ - __xfs_sb_from_disk(&sb, XFS_BUF_TO_SBP(bp), false); + __xfs_sb_from_disk(&sb, dsb, false); error = xfs_validate_sb_common(mp, bp, &sb); if (error) goto out_error; @@ -766,7 +767,7 @@ xfs_sb_write_verify( return; if (bip) - XFS_BUF_TO_SBP(bp)->sb_lsn = cpu_to_be64(bip->bli_item.li_lsn); + dsb->sb_lsn = cpu_to_be64(bip->bli_item.li_lsn); xfs_buf_update_cksum(bp, XFS_SB_CRC_OFF); return; @@ -927,7 +928,7 @@ xfs_log_sb( mp->m_sb.sb_ifree = percpu_counter_sum(&mp->m_ifree); mp->m_sb.sb_fdblocks = percpu_counter_sum(&mp->m_fdblocks); - xfs_sb_to_disk(XFS_BUF_TO_SBP(bp), &mp->m_sb); + xfs_sb_to_disk(bp->b_addr, &mp->m_sb); xfs_trans_buf_set_type(tp, bp, XFS_BLFT_SB_BUF); xfs_trans_log_buf(tp, bp, 0, sizeof(struct xfs_dsb) - 1); } @@ -1007,7 +1008,7 @@ xfs_update_secondary_sbs( bp->b_ops = &xfs_sb_buf_ops; xfs_buf_oneshot(bp); xfs_buf_zero(bp, 0, BBTOB(bp->b_length)); - xfs_sb_to_disk(XFS_BUF_TO_SBP(bp), &mp->m_sb); + xfs_sb_to_disk(bp->b_addr, &mp->m_sb); xfs_buf_delwri_queue(bp, &buffer_list); xfs_buf_relse(bp); diff --git a/fs/xfs/libxfs/xfs_trans_resv.c b/fs/xfs/libxfs/xfs_trans_resv.c index 7a9c04920505..d1a0848cb52e 100644 --- a/fs/xfs/libxfs/xfs_trans_resv.c +++ b/fs/xfs/libxfs/xfs_trans_resv.c @@ -187,7 +187,7 @@ xfs_calc_inode_chunk_res( XFS_FSB_TO_B(mp, 1)); if (alloc) { /* icreate tx uses ordered buffers */ - if (xfs_sb_version_hascrc(&mp->m_sb)) + if (xfs_sb_version_has_v3inode(&mp->m_sb)) return res; size = XFS_FSB_TO_B(mp, 1); } diff --git a/fs/xfs/scrub/agheader.c b/fs/xfs/scrub/agheader.c index ba0f747c82e8..e9bcf1faa183 100644 --- a/fs/xfs/scrub/agheader.c +++ b/fs/xfs/scrub/agheader.c @@ -92,7 +92,7 @@ xchk_superblock( if (!xchk_process_error(sc, agno, XFS_SB_BLOCK(mp), &error)) return error; - sb = XFS_BUF_TO_SBP(bp); + sb = bp->b_addr; /* * Verify the geometries match. Fields that are permanently @@ -358,7 +358,7 @@ static inline void xchk_agf_xref_freeblks( struct xfs_scrub *sc) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(sc->sa.agf_bp); + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; xfs_extlen_t blocks = 0; int error; @@ -378,7 +378,7 @@ static inline void xchk_agf_xref_cntbt( struct xfs_scrub *sc) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(sc->sa.agf_bp); + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; xfs_agblock_t agbno; xfs_extlen_t blocks; int have; @@ -410,7 +410,7 @@ STATIC void xchk_agf_xref_btreeblks( struct xfs_scrub *sc) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(sc->sa.agf_bp); + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; struct xfs_mount *mp = sc->mp; xfs_agblock_t blocks; xfs_agblock_t btreeblks; @@ -456,7 +456,7 @@ static inline void xchk_agf_xref_refcblks( struct xfs_scrub *sc) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(sc->sa.agf_bp); + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; xfs_agblock_t blocks; int error; @@ -525,7 +525,7 @@ xchk_agf( goto out; xchk_buffer_recheck(sc, sc->sa.agf_bp); - agf = XFS_BUF_TO_AGF(sc->sa.agf_bp); + agf = sc->sa.agf_bp->b_addr; /* Check the AG length */ eoag = be32_to_cpu(agf->agf_length); @@ -711,7 +711,7 @@ xchk_agfl( goto out; /* Allocate buffer to ensure uniqueness of AGFL entries. */ - agf = XFS_BUF_TO_AGF(sc->sa.agf_bp); + agf = sc->sa.agf_bp->b_addr; agflcount = be32_to_cpu(agf->agf_flcount); if (agflcount > xfs_agfl_size(sc->mp)) { xchk_block_set_corrupt(sc, sc->sa.agf_bp); @@ -728,7 +728,7 @@ xchk_agfl( } /* Check the blocks in the AGFL. */ - error = xfs_agfl_walk(sc->mp, XFS_BUF_TO_AGF(sc->sa.agf_bp), + error = xfs_agfl_walk(sc->mp, sc->sa.agf_bp->b_addr, sc->sa.agfl_bp, xchk_agfl_block, &sai); if (error == -ECANCELED) { error = 0; @@ -765,7 +765,7 @@ static inline void xchk_agi_xref_icounts( struct xfs_scrub *sc) { - struct xfs_agi *agi = XFS_BUF_TO_AGI(sc->sa.agi_bp); + struct xfs_agi *agi = sc->sa.agi_bp->b_addr; xfs_agino_t icount; xfs_agino_t freecount; int error; @@ -834,7 +834,7 @@ xchk_agi( goto out; xchk_buffer_recheck(sc, sc->sa.agi_bp); - agi = XFS_BUF_TO_AGI(sc->sa.agi_bp); + agi = sc->sa.agi_bp->b_addr; /* Check the AG length */ eoag = be32_to_cpu(agi->agi_length); diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index d5e6db9af434..bca2ab1d4be9 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -49,7 +49,7 @@ xrep_superblock( /* Copy AG 0's superblock to this one. */ xfs_buf_zero(bp, 0, BBTOB(bp->b_length)); - xfs_sb_to_disk(XFS_BUF_TO_SBP(bp), &mp->m_sb); + xfs_sb_to_disk(bp->b_addr, &mp->m_sb); /* Write this to disk. */ xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_SB_BUF); @@ -140,7 +140,7 @@ xrep_agf_find_btrees( struct xrep_find_ag_btree *fab, struct xfs_buf *agfl_bp) { - struct xfs_agf *old_agf = XFS_BUF_TO_AGF(agf_bp); + struct xfs_agf *old_agf = agf_bp->b_addr; int error; /* Go find the root data. */ @@ -181,7 +181,7 @@ xrep_agf_init_header( struct xfs_agf *old_agf) { struct xfs_mount *mp = sc->mp; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agf_bp); + struct xfs_agf *agf = agf_bp->b_addr; memcpy(old_agf, agf, sizeof(*old_agf)); memset(agf, 0, BBTOB(agf_bp->b_length)); @@ -238,7 +238,7 @@ xrep_agf_calc_from_btrees( { struct xrep_agf_allocbt raa = { .sc = sc }; struct xfs_btree_cur *cur = NULL; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agf_bp); + struct xfs_agf *agf = agf_bp->b_addr; struct xfs_mount *mp = sc->mp; xfs_agblock_t btreeblks; xfs_agblock_t blocks; @@ -302,7 +302,7 @@ xrep_agf_commit_new( struct xfs_buf *agf_bp) { struct xfs_perag *pag; - struct xfs_agf *agf = XFS_BUF_TO_AGF(agf_bp); + struct xfs_agf *agf = agf_bp->b_addr; /* Trigger fdblocks recalculation */ xfs_force_summary_recalc(sc->mp); @@ -376,7 +376,7 @@ xrep_agf( if (error) return error; agf_bp->b_ops = &xfs_agf_buf_ops; - agf = XFS_BUF_TO_AGF(agf_bp); + agf = agf_bp->b_addr; /* * Load the AGFL so that we can screen out OWN_AG blocks that are on @@ -395,7 +395,7 @@ xrep_agf( * Spot-check the AGFL blocks; if they're obviously corrupt then * there's nothing we can do but bail out. */ - error = xfs_agfl_walk(sc->mp, XFS_BUF_TO_AGF(agf_bp), agfl_bp, + error = xfs_agfl_walk(sc->mp, agf_bp->b_addr, agfl_bp, xrep_agf_check_agfl_block, sc); if (error) return error; @@ -429,10 +429,10 @@ xrep_agf( struct xrep_agfl { /* Bitmap of other OWN_AG metadata blocks. */ - struct xfs_bitmap agmetablocks; + struct xbitmap agmetablocks; /* Bitmap of free space. */ - struct xfs_bitmap *freesp; + struct xbitmap *freesp; struct xfs_scrub *sc; }; @@ -453,14 +453,14 @@ xrep_agfl_walk_rmap( /* Record all the OWN_AG blocks. */ if (rec->rm_owner == XFS_RMAP_OWN_AG) { - fsb = XFS_AGB_TO_FSB(cur->bc_mp, cur->bc_private.a.agno, + fsb = XFS_AGB_TO_FSB(cur->bc_mp, cur->bc_ag.agno, rec->rm_startblock); - error = xfs_bitmap_set(ra->freesp, fsb, rec->rm_blockcount); + error = xbitmap_set(ra->freesp, fsb, rec->rm_blockcount); if (error) return error; } - return xfs_bitmap_set_btcur_path(&ra->agmetablocks, cur); + return xbitmap_set_btcur_path(&ra->agmetablocks, cur); } /* @@ -476,19 +476,17 @@ STATIC int xrep_agfl_collect_blocks( struct xfs_scrub *sc, struct xfs_buf *agf_bp, - struct xfs_bitmap *agfl_extents, + struct xbitmap *agfl_extents, xfs_agblock_t *flcount) { struct xrep_agfl ra; struct xfs_mount *mp = sc->mp; struct xfs_btree_cur *cur; - struct xfs_bitmap_range *br; - struct xfs_bitmap_range *n; int error; ra.sc = sc; ra.freesp = agfl_extents; - xfs_bitmap_init(&ra.agmetablocks); + xbitmap_init(&ra.agmetablocks); /* Find all space used by the free space btrees & rmapbt. */ cur = xfs_rmapbt_init_cursor(mp, sc->tp, agf_bp, sc->sa.agno); @@ -500,7 +498,7 @@ xrep_agfl_collect_blocks( /* Find all blocks currently being used by the bnobt. */ cur = xfs_allocbt_init_cursor(mp, sc->tp, agf_bp, sc->sa.agno, XFS_BTNUM_BNO); - error = xfs_bitmap_set_btblocks(&ra.agmetablocks, cur); + error = xbitmap_set_btblocks(&ra.agmetablocks, cur); if (error) goto err; xfs_btree_del_cursor(cur, error); @@ -508,7 +506,7 @@ xrep_agfl_collect_blocks( /* Find all blocks currently being used by the cntbt. */ cur = xfs_allocbt_init_cursor(mp, sc->tp, agf_bp, sc->sa.agno, XFS_BTNUM_CNT); - error = xfs_bitmap_set_btblocks(&ra.agmetablocks, cur); + error = xbitmap_set_btblocks(&ra.agmetablocks, cur); if (error) goto err; @@ -518,8 +516,8 @@ xrep_agfl_collect_blocks( * Drop the freesp meta blocks that are in use by btrees. * The remaining blocks /should/ be AGFL blocks. */ - error = xfs_bitmap_disunion(agfl_extents, &ra.agmetablocks); - xfs_bitmap_destroy(&ra.agmetablocks); + error = xbitmap_disunion(agfl_extents, &ra.agmetablocks); + xbitmap_destroy(&ra.agmetablocks); if (error) return error; @@ -527,18 +525,12 @@ xrep_agfl_collect_blocks( * Calculate the new AGFL size. If we found more blocks than fit in * the AGFL we'll free them later. */ - *flcount = 0; - for_each_xfs_bitmap_extent(br, n, agfl_extents) { - *flcount += br->len; - if (*flcount > xfs_agfl_size(mp)) - break; - } - if (*flcount > xfs_agfl_size(mp)) - *flcount = xfs_agfl_size(mp); + *flcount = min_t(uint64_t, xbitmap_hweight(agfl_extents), + xfs_agfl_size(mp)); return 0; err: - xfs_bitmap_destroy(&ra.agmetablocks); + xbitmap_destroy(&ra.agmetablocks); xfs_btree_del_cursor(cur, error); return error; } @@ -550,7 +542,7 @@ xrep_agfl_update_agf( struct xfs_buf *agf_bp, xfs_agblock_t flcount) { - struct xfs_agf *agf = XFS_BUF_TO_AGF(agf_bp); + struct xfs_agf *agf = agf_bp->b_addr; ASSERT(flcount <= xfs_agfl_size(sc->mp)); @@ -573,13 +565,13 @@ STATIC void xrep_agfl_init_header( struct xfs_scrub *sc, struct xfs_buf *agfl_bp, - struct xfs_bitmap *agfl_extents, + struct xbitmap *agfl_extents, xfs_agblock_t flcount) { struct xfs_mount *mp = sc->mp; __be32 *agfl_bno; - struct xfs_bitmap_range *br; - struct xfs_bitmap_range *n; + struct xbitmap_range *br; + struct xbitmap_range *n; struct xfs_agfl *agfl; xfs_agblock_t agbno; unsigned int fl_off; @@ -602,8 +594,8 @@ xrep_agfl_init_header( * step. */ fl_off = 0; - agfl_bno = XFS_BUF_TO_AGFL_BNO(mp, agfl_bp); - for_each_xfs_bitmap_extent(br, n, agfl_extents) { + agfl_bno = xfs_buf_to_agfl_bno(agfl_bp); + for_each_xbitmap_extent(br, n, agfl_extents) { agbno = XFS_FSB_TO_AGBNO(mp, br->start); trace_xrep_agfl_insert(mp, sc->sa.agno, agbno, br->len); @@ -637,7 +629,7 @@ int xrep_agfl( struct xfs_scrub *sc) { - struct xfs_bitmap agfl_extents; + struct xbitmap agfl_extents; struct xfs_mount *mp = sc->mp; struct xfs_buf *agf_bp; struct xfs_buf *agfl_bp; @@ -649,7 +641,7 @@ xrep_agfl( return -EOPNOTSUPP; xchk_perag_get(sc->mp, &sc->sa); - xfs_bitmap_init(&agfl_extents); + xbitmap_init(&agfl_extents); /* * Read the AGF so that we can query the rmapbt. We hope that there's @@ -696,10 +688,10 @@ xrep_agfl( goto err; /* Dump any AGFL overflow. */ - return xrep_reap_extents(sc, &agfl_extents, &XFS_RMAP_OINFO_AG, + error = xrep_reap_extents(sc, &agfl_extents, &XFS_RMAP_OINFO_AG, XFS_AG_RESV_AGFL); err: - xfs_bitmap_destroy(&agfl_extents); + xbitmap_destroy(&agfl_extents); return error; } @@ -761,7 +753,7 @@ xrep_agi_init_header( struct xfs_buf *agi_bp, struct xfs_agi *old_agi) { - struct xfs_agi *agi = XFS_BUF_TO_AGI(agi_bp); + struct xfs_agi *agi = agi_bp->b_addr; struct xfs_mount *mp = sc->mp; memcpy(old_agi, agi, sizeof(*old_agi)); @@ -807,7 +799,7 @@ xrep_agi_calc_from_btrees( struct xfs_buf *agi_bp) { struct xfs_btree_cur *cur; - struct xfs_agi *agi = XFS_BUF_TO_AGI(agi_bp); + struct xfs_agi *agi = agi_bp->b_addr; struct xfs_mount *mp = sc->mp; xfs_agino_t count; xfs_agino_t freecount; @@ -835,7 +827,7 @@ xrep_agi_commit_new( struct xfs_buf *agi_bp) { struct xfs_perag *pag; - struct xfs_agi *agi = XFS_BUF_TO_AGI(agi_bp); + struct xfs_agi *agi = agi_bp->b_addr; /* Trigger inode count recalculation */ xfs_force_summary_recalc(sc->mp); @@ -892,7 +884,7 @@ xrep_agi( if (error) return error; agi_bp->b_ops = &xfs_agi_buf_ops; - agi = XFS_BUF_TO_AGI(agi_bp); + agi = agi_bp->b_addr; /* Find the AGI btree roots. */ error = xrep_agi_find_btrees(sc, fab); diff --git a/fs/xfs/scrub/alloc.c b/fs/xfs/scrub/alloc.c index 5533e48e605d..73d924e47565 100644 --- a/fs/xfs/scrub/alloc.c +++ b/fs/xfs/scrub/alloc.c @@ -94,7 +94,7 @@ xchk_allocbt_rec( union xfs_btree_rec *rec) { struct xfs_mount *mp = bs->cur->bc_mp; - xfs_agnumber_t agno = bs->cur->bc_private.a.agno; + xfs_agnumber_t agno = bs->cur->bc_ag.agno; xfs_agblock_t bno; xfs_extlen_t len; diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index d9f0dd444b80..9faddb334a2c 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -98,7 +98,7 @@ struct xchk_xattr { /* * Check that an extended attribute key can be looked up by hash. * - * We use the XFS attribute list iterator (i.e. xfs_attr_list_int_ilocked) + * We use the XFS attribute list iterator (i.e. xfs_attr_list_ilocked) * to call this function for every attribute key in an inode. Once * we're here, we load the attribute value to see if any errors happen, * or if we get more or less data than we expected. @@ -147,11 +147,8 @@ xchk_xattr_listent( return; } - args.flags = ATTR_KERNOTIME; - if (flags & XFS_ATTR_ROOT) - args.flags |= ATTR_ROOT; - else if (flags & XFS_ATTR_SECURE) - args.flags |= ATTR_SECURE; + args.op_flags = XFS_DA_OP_NOTIME; + args.attr_filter = flags & XFS_ATTR_NSP_ONDISK_MASK; args.geo = context->dp->i_mount->m_attr_geo; args.whichfork = XFS_ATTR_FORK; args.dp = context->dp; @@ -162,7 +159,10 @@ xchk_xattr_listent( args.value = xchk_xattr_valuebuf(sx->sc); args.valuelen = valuelen; - error = xfs_attr_get_ilocked(context->dp, &args); + error = xfs_attr_get_ilocked(&args); + /* ENODATA means the hash lookup failed and the attr is bad */ + if (error == -ENODATA) + error = -EFSCORRUPTED; if (!xchk_fblock_process_error(sx->sc, XFS_ATTR_FORK, args.blkno, &error)) goto fail_xref; @@ -474,7 +474,6 @@ xchk_xattr( struct xfs_scrub *sc) { struct xchk_xattr sx; - struct attrlist_cursor_kern cursor = { 0 }; xfs_dablk_t last_checked = -1U; int error = 0; @@ -493,11 +492,10 @@ xchk_xattr( /* Check that every attr key can also be looked up by hash. */ sx.context.dp = sc->ip; - sx.context.cursor = &cursor; sx.context.resynch = 1; sx.context.put_listent = xchk_xattr_listent; sx.context.tp = sc->tp; - sx.context.flags = ATTR_INCOMPLETE; + sx.context.allow_incomplete = true; sx.sc = sc; /* @@ -516,7 +514,7 @@ xchk_xattr( * iteration, which doesn't really follow the usual buffer * locking order. */ - error = xfs_attr_list_int_ilocked(&sx.context); + error = xfs_attr_list_ilocked(&sx.context); if (!xchk_fblock_process_error(sc, XFS_ATTR_FORK, 0, &error)) goto out; diff --git a/fs/xfs/scrub/bitmap.c b/fs/xfs/scrub/bitmap.c index 18a684e18a69..f88694f22d05 100644 --- a/fs/xfs/scrub/bitmap.c +++ b/fs/xfs/scrub/bitmap.c @@ -18,14 +18,14 @@ * This is the logical equivalent of bitmap |= mask(start, len). */ int -xfs_bitmap_set( - struct xfs_bitmap *bitmap, +xbitmap_set( + struct xbitmap *bitmap, uint64_t start, uint64_t len) { - struct xfs_bitmap_range *bmr; + struct xbitmap_range *bmr; - bmr = kmem_alloc(sizeof(struct xfs_bitmap_range), KM_MAYFAIL); + bmr = kmem_alloc(sizeof(struct xbitmap_range), KM_MAYFAIL); if (!bmr) return -ENOMEM; @@ -39,13 +39,13 @@ xfs_bitmap_set( /* Free everything related to this bitmap. */ void -xfs_bitmap_destroy( - struct xfs_bitmap *bitmap) +xbitmap_destroy( + struct xbitmap *bitmap) { - struct xfs_bitmap_range *bmr; - struct xfs_bitmap_range *n; + struct xbitmap_range *bmr; + struct xbitmap_range *n; - for_each_xfs_bitmap_extent(bmr, n, bitmap) { + for_each_xbitmap_extent(bmr, n, bitmap) { list_del(&bmr->list); kmem_free(bmr); } @@ -53,24 +53,24 @@ xfs_bitmap_destroy( /* Set up a per-AG block bitmap. */ void -xfs_bitmap_init( - struct xfs_bitmap *bitmap) +xbitmap_init( + struct xbitmap *bitmap) { INIT_LIST_HEAD(&bitmap->list); } /* Compare two btree extents. */ static int -xfs_bitmap_range_cmp( +xbitmap_range_cmp( void *priv, struct list_head *a, struct list_head *b) { - struct xfs_bitmap_range *ap; - struct xfs_bitmap_range *bp; + struct xbitmap_range *ap; + struct xbitmap_range *bp; - ap = container_of(a, struct xfs_bitmap_range, list); - bp = container_of(b, struct xfs_bitmap_range, list); + ap = container_of(a, struct xbitmap_range, list); + bp = container_of(b, struct xbitmap_range, list); if (ap->start > bp->start) return 1; @@ -96,14 +96,14 @@ xfs_bitmap_range_cmp( #define LEFT_ALIGNED (1 << 0) #define RIGHT_ALIGNED (1 << 1) int -xfs_bitmap_disunion( - struct xfs_bitmap *bitmap, - struct xfs_bitmap *sub) +xbitmap_disunion( + struct xbitmap *bitmap, + struct xbitmap *sub) { struct list_head *lp; - struct xfs_bitmap_range *br; - struct xfs_bitmap_range *new_br; - struct xfs_bitmap_range *sub_br; + struct xbitmap_range *br; + struct xbitmap_range *new_br; + struct xbitmap_range *sub_br; uint64_t sub_start; uint64_t sub_len; int state; @@ -113,8 +113,8 @@ xfs_bitmap_disunion( return 0; ASSERT(!list_empty(&sub->list)); - list_sort(NULL, &bitmap->list, xfs_bitmap_range_cmp); - list_sort(NULL, &sub->list, xfs_bitmap_range_cmp); + list_sort(NULL, &bitmap->list, xbitmap_range_cmp); + list_sort(NULL, &sub->list, xbitmap_range_cmp); /* * Now that we've sorted both lists, we iterate bitmap once, rolling @@ -124,11 +124,11 @@ xfs_bitmap_disunion( * list traversal is similar to merge sort, but we're deleting * instead. In this manner we avoid O(n^2) operations. */ - sub_br = list_first_entry(&sub->list, struct xfs_bitmap_range, + sub_br = list_first_entry(&sub->list, struct xbitmap_range, list); lp = bitmap->list.next; while (lp != &bitmap->list) { - br = list_entry(lp, struct xfs_bitmap_range, list); + br = list_entry(lp, struct xbitmap_range, list); /* * Advance sub_br and/or br until we find a pair that @@ -181,7 +181,7 @@ xfs_bitmap_disunion( * Deleting from the middle: add the new right extent * and then shrink the left extent. */ - new_br = kmem_alloc(sizeof(struct xfs_bitmap_range), + new_br = kmem_alloc(sizeof(struct xbitmap_range), KM_MAYFAIL); if (!new_br) { error = -ENOMEM; @@ -247,8 +247,8 @@ xfs_bitmap_disunion( * blocks going from the leaf towards the root. */ int -xfs_bitmap_set_btcur_path( - struct xfs_bitmap *bitmap, +xbitmap_set_btcur_path( + struct xbitmap *bitmap, struct xfs_btree_cur *cur) { struct xfs_buf *bp; @@ -261,7 +261,7 @@ xfs_bitmap_set_btcur_path( if (!bp) continue; fsb = XFS_DADDR_TO_FSB(cur->bc_mp, bp->b_bn); - error = xfs_bitmap_set(bitmap, fsb, 1); + error = xbitmap_set(bitmap, fsb, 1); if (error) return error; } @@ -271,12 +271,12 @@ xfs_bitmap_set_btcur_path( /* Collect a btree's block in the bitmap. */ STATIC int -xfs_bitmap_collect_btblock( +xbitmap_collect_btblock( struct xfs_btree_cur *cur, int level, void *priv) { - struct xfs_bitmap *bitmap = priv; + struct xbitmap *bitmap = priv; struct xfs_buf *bp; xfs_fsblock_t fsbno; @@ -285,15 +285,30 @@ xfs_bitmap_collect_btblock( return 0; fsbno = XFS_DADDR_TO_FSB(cur->bc_mp, bp->b_bn); - return xfs_bitmap_set(bitmap, fsbno, 1); + return xbitmap_set(bitmap, fsbno, 1); } /* Walk the btree and mark the bitmap wherever a btree block is found. */ int -xfs_bitmap_set_btblocks( - struct xfs_bitmap *bitmap, +xbitmap_set_btblocks( + struct xbitmap *bitmap, struct xfs_btree_cur *cur) { - return xfs_btree_visit_blocks(cur, xfs_bitmap_collect_btblock, + return xfs_btree_visit_blocks(cur, xbitmap_collect_btblock, XFS_BTREE_VISIT_ALL, bitmap); } + +/* How many bits are set in this bitmap? */ +uint64_t +xbitmap_hweight( + struct xbitmap *bitmap) +{ + struct xbitmap_range *bmr; + struct xbitmap_range *n; + uint64_t ret = 0; + + for_each_xbitmap_extent(bmr, n, bitmap) + ret += bmr->len; + + return ret; +} diff --git a/fs/xfs/scrub/bitmap.h b/fs/xfs/scrub/bitmap.h index ae8ecbce6fa6..900646b72de1 100644 --- a/fs/xfs/scrub/bitmap.h +++ b/fs/xfs/scrub/bitmap.h @@ -6,31 +6,32 @@ #ifndef __XFS_SCRUB_BITMAP_H__ #define __XFS_SCRUB_BITMAP_H__ -struct xfs_bitmap_range { +struct xbitmap_range { struct list_head list; uint64_t start; uint64_t len; }; -struct xfs_bitmap { +struct xbitmap { struct list_head list; }; -void xfs_bitmap_init(struct xfs_bitmap *bitmap); -void xfs_bitmap_destroy(struct xfs_bitmap *bitmap); +void xbitmap_init(struct xbitmap *bitmap); +void xbitmap_destroy(struct xbitmap *bitmap); -#define for_each_xfs_bitmap_extent(bex, n, bitmap) \ +#define for_each_xbitmap_extent(bex, n, bitmap) \ list_for_each_entry_safe((bex), (n), &(bitmap)->list, list) -#define for_each_xfs_bitmap_block(b, bex, n, bitmap) \ +#define for_each_xbitmap_block(b, bex, n, bitmap) \ list_for_each_entry_safe((bex), (n), &(bitmap)->list, list) \ - for ((b) = bex->start; (b) < bex->start + bex->len; (b)++) + for ((b) = (bex)->start; (b) < (bex)->start + (bex)->len; (b)++) -int xfs_bitmap_set(struct xfs_bitmap *bitmap, uint64_t start, uint64_t len); -int xfs_bitmap_disunion(struct xfs_bitmap *bitmap, struct xfs_bitmap *sub); -int xfs_bitmap_set_btcur_path(struct xfs_bitmap *bitmap, +int xbitmap_set(struct xbitmap *bitmap, uint64_t start, uint64_t len); +int xbitmap_disunion(struct xbitmap *bitmap, struct xbitmap *sub); +int xbitmap_set_btcur_path(struct xbitmap *bitmap, struct xfs_btree_cur *cur); -int xfs_bitmap_set_btblocks(struct xfs_bitmap *bitmap, +int xbitmap_set_btblocks(struct xbitmap *bitmap, struct xfs_btree_cur *cur); +uint64_t xbitmap_hweight(struct xbitmap *bitmap); #endif /* __XFS_SCRUB_BITMAP_H__ */ diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c index fa6ea6407992..add8598eacd5 100644 --- a/fs/xfs/scrub/bmap.c +++ b/fs/xfs/scrub/bmap.c @@ -374,7 +374,7 @@ xchk_bmapbt_rec( struct xfs_bmbt_irec iext_irec; struct xfs_iext_cursor icur; struct xchk_bmap_info *info = bs->private; - struct xfs_inode *ip = bs->cur->bc_private.b.ip; + struct xfs_inode *ip = bs->cur->bc_ino.ip; struct xfs_buf *bp = NULL; struct xfs_btree_block *block; struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, info->whichfork); @@ -501,7 +501,7 @@ xchk_bmap_check_rmap( xchk_fblock_set_corrupt(sc, sbcri->whichfork, rec->rm_offset); if (irec.br_startblock != XFS_AGB_TO_FSB(sc->mp, - cur->bc_private.a.agno, rec->rm_startblock)) + cur->bc_ag.agno, rec->rm_startblock)) xchk_fblock_set_corrupt(sc, sbcri->whichfork, rec->rm_offset); if (irec.br_blockcount > rec->rm_blockcount) diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index 97a15b6f2865..9a2e27ac1300 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -219,19 +219,21 @@ xchk_da_btree_block_check_sibling( int direction, xfs_dablk_t sibling) { + struct xfs_da_state_path *path = &ds->state->path; + struct xfs_da_state_path *altpath = &ds->state->altpath; int retval; + int plevel; int error; - memcpy(&ds->state->altpath, &ds->state->path, - sizeof(ds->state->altpath)); + memcpy(altpath, path, sizeof(ds->state->altpath)); /* * If the pointer is null, we shouldn't be able to move the upper * level pointer anywhere. */ if (sibling == 0) { - error = xfs_da3_path_shift(ds->state, &ds->state->altpath, - direction, false, &retval); + error = xfs_da3_path_shift(ds->state, altpath, direction, + false, &retval); if (error == 0 && retval == 0) xchk_da_set_corrupt(ds, level); error = 0; @@ -239,27 +241,33 @@ xchk_da_btree_block_check_sibling( } /* Move the alternate cursor one block in the direction given. */ - error = xfs_da3_path_shift(ds->state, &ds->state->altpath, - direction, false, &retval); + error = xfs_da3_path_shift(ds->state, altpath, direction, false, + &retval); if (!xchk_da_process_error(ds, level, &error)) - return error; + goto out; if (retval) { xchk_da_set_corrupt(ds, level); - return error; + goto out; } - if (ds->state->altpath.blk[level].bp) - xchk_buffer_recheck(ds->sc, - ds->state->altpath.blk[level].bp); + if (altpath->blk[level].bp) + xchk_buffer_recheck(ds->sc, altpath->blk[level].bp); /* Compare upper level pointer to sibling pointer. */ - if (ds->state->altpath.blk[level].blkno != sibling) + if (altpath->blk[level].blkno != sibling) xchk_da_set_corrupt(ds, level); - if (ds->state->altpath.blk[level].bp) { - xfs_trans_brelse(ds->dargs.trans, - ds->state->altpath.blk[level].bp); - ds->state->altpath.blk[level].bp = NULL; - } + out: + /* Free all buffers in the altpath that aren't referenced from path. */ + for (plevel = 0; plevel < altpath->active; plevel++) { + if (altpath->blk[plevel].bp == NULL || + (plevel < path->active && + altpath->blk[plevel].bp == path->blk[plevel].bp)) + continue; + + xfs_trans_brelse(ds->dargs.trans, altpath->blk[plevel].bp); + altpath->blk[plevel].bp = NULL; + } + return error; } diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 266da4e4bde6..fe2a6e030c8a 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -155,6 +155,9 @@ xchk_dir_actor( xname.type = XFS_DIR3_FT_UNKNOWN; error = xfs_dir_lookup(sdc->sc->tp, ip, &xname, &lookup_ino, NULL); + /* ENOENT means the hash lookup failed and the dir is corrupt */ + if (error == -ENOENT) + error = -EFSCORRUPTED; if (!xchk_fblock_process_error(sdc->sc, XFS_DATA_FORK, offset, &error)) goto out; @@ -500,7 +503,7 @@ xchk_directory_leaf1_bestfree( /* Read the free space block. */ error = xfs_dir3_leaf_read(sc->tp, sc->ip, lblk, &bp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) - goto out; + return error; xchk_buffer_recheck(sc, bp); leaf = bp->b_addr; @@ -565,9 +568,10 @@ xchk_directory_leaf1_bestfree( xchk_directory_check_freesp(sc, lblk, dbp, best); xfs_trans_brelse(sc->tp, dbp); if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) - goto out; + break; } out: + xfs_trans_brelse(sc->tp, bp); return error; } @@ -589,7 +593,7 @@ xchk_directory_free_bestfree( /* Read the free space block */ error = xfs_dir2_free_read(sc->tp, sc->ip, lblk, &bp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) - goto out; + return error; xchk_buffer_recheck(sc, bp); if (xfs_sb_version_hascrc(&sc->mp->m_sb)) { @@ -612,7 +616,7 @@ xchk_directory_free_bestfree( 0, &dbp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) - break; + goto out; xchk_directory_check_freesp(sc, lblk, dbp, best); xfs_trans_brelse(sc->tp, dbp); } @@ -620,6 +624,7 @@ xchk_directory_free_bestfree( if (freehdr.nused + stale != freehdr.nvalid) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); out: + xfs_trans_brelse(sc->tp, bp); return error; } diff --git a/fs/xfs/scrub/ialloc.c b/fs/xfs/scrub/ialloc.c index 681758704fda..64c217eb06a7 100644 --- a/fs/xfs/scrub/ialloc.c +++ b/fs/xfs/scrub/ialloc.c @@ -104,7 +104,7 @@ xchk_iallocbt_chunk( xfs_extlen_t len) { struct xfs_mount *mp = bs->cur->bc_mp; - xfs_agnumber_t agno = bs->cur->bc_private.a.agno; + xfs_agnumber_t agno = bs->cur->bc_ag.agno; xfs_agblock_t bno; bno = XFS_AGINO_TO_AGBNO(mp, agino); @@ -164,7 +164,7 @@ xchk_iallocbt_check_cluster_ifree( * the record, compute which fs inode we're talking about. */ agino = irec->ir_startino + irec_ino; - fsino = XFS_AGINO_TO_INO(mp, bs->cur->bc_private.a.agno, agino); + fsino = XFS_AGINO_TO_INO(mp, bs->cur->bc_ag.agno, agino); irec_free = (irec->ir_free & XFS_INOBT_MASK(irec_ino)); if (be16_to_cpu(dip->di_magic) != XFS_DINODE_MAGIC || @@ -215,7 +215,7 @@ xchk_iallocbt_check_cluster( struct xfs_dinode *dip; struct xfs_buf *cluster_bp; unsigned int nr_inodes; - xfs_agnumber_t agno = bs->cur->bc_private.a.agno; + xfs_agnumber_t agno = bs->cur->bc_ag.agno; xfs_agblock_t agbno; unsigned int cluster_index; uint16_t cluster_mask = 0; @@ -426,7 +426,7 @@ xchk_iallocbt_rec( struct xchk_iallocbt *iabt = bs->private; struct xfs_inobt_rec_incore irec; uint64_t holes; - xfs_agnumber_t agno = bs->cur->bc_private.a.agno; + xfs_agnumber_t agno = bs->cur->bc_ag.agno; xfs_agino_t agino; xfs_extlen_t len; int holecount; diff --git a/fs/xfs/scrub/refcount.c b/fs/xfs/scrub/refcount.c index 0cab11a5d390..beaeb6fa3119 100644 --- a/fs/xfs/scrub/refcount.c +++ b/fs/xfs/scrub/refcount.c @@ -336,7 +336,7 @@ xchk_refcountbt_rec( { struct xfs_mount *mp = bs->cur->bc_mp; xfs_agblock_t *cow_blocks = bs->private; - xfs_agnumber_t agno = bs->cur->bc_private.a.agno; + xfs_agnumber_t agno = bs->cur->bc_ag.agno; xfs_agblock_t bno; xfs_extlen_t len; xfs_nlink_t refcount; diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index e489d7a8446a..db3cfd12803d 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -208,8 +208,10 @@ xrep_calc_ag_resblks( /* Now grab the block counters from the AGF. */ error = xfs_alloc_read_agf(mp, NULL, sm->sm_agno, 0, &bp); if (!error) { - aglen = be32_to_cpu(XFS_BUF_TO_AGF(bp)->agf_length); - freelen = be32_to_cpu(XFS_BUF_TO_AGF(bp)->agf_freeblks); + struct xfs_agf *agf = bp->b_addr; + + aglen = be32_to_cpu(agf->agf_length); + freelen = be32_to_cpu(agf->agf_freeblks); usedlen = aglen - freelen; xfs_buf_relse(bp); } @@ -434,10 +436,10 @@ xrep_init_btblock( int xrep_invalidate_blocks( struct xfs_scrub *sc, - struct xfs_bitmap *bitmap) + struct xbitmap *bitmap) { - struct xfs_bitmap_range *bmr; - struct xfs_bitmap_range *n; + struct xbitmap_range *bmr; + struct xbitmap_range *n; struct xfs_buf *bp; xfs_fsblock_t fsbno; @@ -449,7 +451,7 @@ xrep_invalidate_blocks( * because we never own those; and if we can't TRYLOCK the buffer we * assume it's owned by someone else. */ - for_each_xfs_bitmap_block(fsbno, bmr, n, bitmap) { + for_each_xbitmap_block(fsbno, bmr, n, bitmap) { /* Skip AG headers and post-EOFS blocks */ if (!xfs_verify_fsbno(sc->mp, fsbno)) continue; @@ -595,18 +597,18 @@ xrep_reap_block( int xrep_reap_extents( struct xfs_scrub *sc, - struct xfs_bitmap *bitmap, + struct xbitmap *bitmap, const struct xfs_owner_info *oinfo, enum xfs_ag_resv_type type) { - struct xfs_bitmap_range *bmr; - struct xfs_bitmap_range *n; + struct xbitmap_range *bmr; + struct xbitmap_range *n; xfs_fsblock_t fsbno; int error = 0; ASSERT(xfs_sb_version_hasrmapbt(&sc->mp->m_sb)); - for_each_xfs_bitmap_block(fsbno, bmr, n, bitmap) { + for_each_xbitmap_block(fsbno, bmr, n, bitmap) { ASSERT(sc->ip != NULL || XFS_FSB_TO_AGNO(sc->mp, fsbno) == sc->sa.agno); trace_xrep_dispose_btree_extent(sc->mp, @@ -615,11 +617,9 @@ xrep_reap_extents( error = xrep_reap_block(sc, fsbno, oinfo, type); if (error) - goto out; + break; } -out: - xfs_bitmap_destroy(bitmap); return error; } @@ -879,7 +879,7 @@ xrep_find_ag_btree_roots( ri.sc = sc; ri.btree_info = btree_info; - ri.agf = XFS_BUF_TO_AGF(agf_bp); + ri.agf = agf_bp->b_addr; ri.agfl_bp = agfl_bp; for (fab = btree_info; fab->buf_ops; fab++) { ASSERT(agfl_bp || fab->rmap_owner != XFS_RMAP_OWN_AG); diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h index c3422403b169..04a47d45605b 100644 --- a/fs/xfs/scrub/repair.h +++ b/fs/xfs/scrub/repair.h @@ -28,11 +28,11 @@ int xrep_init_btblock(struct xfs_scrub *sc, xfs_fsblock_t fsb, struct xfs_buf **bpp, xfs_btnum_t btnum, const struct xfs_buf_ops *ops); -struct xfs_bitmap; +struct xbitmap; int xrep_fix_freelist(struct xfs_scrub *sc, bool can_shrink); -int xrep_invalidate_blocks(struct xfs_scrub *sc, struct xfs_bitmap *btlist); -int xrep_reap_extents(struct xfs_scrub *sc, struct xfs_bitmap *exlist, +int xrep_invalidate_blocks(struct xfs_scrub *sc, struct xbitmap *btlist); +int xrep_reap_extents(struct xfs_scrub *sc, struct xbitmap *exlist, const struct xfs_owner_info *oinfo, enum xfs_ag_resv_type type); struct xrep_find_ag_btree { diff --git a/fs/xfs/scrub/rmap.c b/fs/xfs/scrub/rmap.c index 8d4cefd761c1..f4fcb4719f41 100644 --- a/fs/xfs/scrub/rmap.c +++ b/fs/xfs/scrub/rmap.c @@ -92,7 +92,7 @@ xchk_rmapbt_rec( { struct xfs_mount *mp = bs->cur->bc_mp; struct xfs_rmap_irec irec; - xfs_agnumber_t agno = bs->cur->bc_private.a.agno; + xfs_agnumber_t agno = bs->cur->bc_ag.agno; bool non_inode; bool is_unwritten; bool is_bmbt; diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index f1775bb19313..8ebf35b115ce 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -168,6 +168,7 @@ xchk_teardown( xfs_irele(sc->ip); sc->ip = NULL; } + sb_end_write(sc->mp->m_super); if (sc->flags & XCHK_REAPING_DISABLED) xchk_start_reaping(sc); if (sc->flags & XCHK_HAS_QUOTAOFFLOCK) { @@ -490,6 +491,14 @@ xfs_scrub_metadata( sc.ops = &meta_scrub_ops[sm->sm_type]; sc.sick_mask = xchk_health_mask_for_scrub_type(sm->sm_type); retry_op: + /* + * If freeze runs concurrently with a scrub, the freeze can be delayed + * indefinitely as we walk the filesystem and iterate over metadata + * buffers. Freeze quiesces the log (which waits for the buffer LRU to + * be emptied) and that won't happen while checking is running. + */ + sb_start_write(mp->m_super); + /* Set up for the operation. */ error = sc.ops->setup(&sc, ip); if (error) diff --git a/fs/xfs/scrub/trace.c b/fs/xfs/scrub/trace.c index 9eaab2eb5ed3..2c6c248be823 100644 --- a/fs/xfs/scrub/trace.c +++ b/fs/xfs/scrub/trace.c @@ -24,9 +24,9 @@ xchk_btree_cur_fsbno( return XFS_DADDR_TO_FSB(cur->bc_mp, cur->bc_bufs[level]->b_bn); else if (level == cur->bc_nlevels - 1 && cur->bc_flags & XFS_BTREE_LONG_PTRS) - return XFS_INO_TO_FSB(cur->bc_mp, cur->bc_private.b.ip->i_ino); + return XFS_INO_TO_FSB(cur->bc_mp, cur->bc_ino.ip->i_ino); else if (!(cur->bc_flags & XFS_BTREE_LONG_PTRS)) - return XFS_AGB_TO_FSB(cur->bc_mp, cur->bc_private.a.agno, 0); + return XFS_AGB_TO_FSB(cur->bc_mp, cur->bc_ag.agno, 0); return NULLFSBLOCK; } diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 096203119934..e46f5cef90da 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -379,7 +379,7 @@ TRACE_EVENT(xchk_ifork_btree_op_error, xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); __entry->dev = sc->mp->m_super->s_dev; __entry->ino = sc->ip->i_ino; - __entry->whichfork = cur->bc_private.b.whichfork; + __entry->whichfork = cur->bc_ino.whichfork; __entry->type = sc->sm->sm_type; __entry->btnum = cur->bc_btnum; __entry->level = level; @@ -459,7 +459,7 @@ TRACE_EVENT(xchk_ifork_btree_error, xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); __entry->dev = sc->mp->m_super->s_dev; __entry->ino = sc->ip->i_ino; - __entry->whichfork = cur->bc_private.b.whichfork; + __entry->whichfork = cur->bc_ino.whichfork; __entry->type = sc->sm->sm_type; __entry->btnum = cur->bc_btnum; __entry->level = level; diff --git a/fs/xfs/xfs_acl.c b/fs/xfs/xfs_acl.c index cd743fad8478..d4c687b5cd06 100644 --- a/fs/xfs/xfs_acl.c +++ b/fs/xfs/xfs_acl.c @@ -14,6 +14,8 @@ #include "xfs_trace.h" #include "xfs_error.h" #include "xfs_acl.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" #include @@ -67,10 +69,12 @@ xfs_acl_from_disk( switch (acl_e->e_tag) { case ACL_USER: - acl_e->e_uid = xfs_uid_to_kuid(be32_to_cpu(ace->ae_id)); + acl_e->e_uid = make_kuid(&init_user_ns, + be32_to_cpu(ace->ae_id)); break; case ACL_GROUP: - acl_e->e_gid = xfs_gid_to_kgid(be32_to_cpu(ace->ae_id)); + acl_e->e_gid = make_kgid(&init_user_ns, + be32_to_cpu(ace->ae_id)); break; case ACL_USER_OBJ: case ACL_GROUP_OBJ: @@ -103,10 +107,12 @@ xfs_acl_to_disk(struct xfs_acl *aclp, const struct posix_acl *acl) ace->ae_tag = cpu_to_be32(acl_e->e_tag); switch (acl_e->e_tag) { case ACL_USER: - ace->ae_id = cpu_to_be32(xfs_kuid_to_uid(acl_e->e_uid)); + ace->ae_id = cpu_to_be32( + from_kuid(&init_user_ns, acl_e->e_uid)); break; case ACL_GROUP: - ace->ae_id = cpu_to_be32(xfs_kgid_to_gid(acl_e->e_gid)); + ace->ae_id = cpu_to_be32( + from_kgid(&init_user_ns, acl_e->e_gid)); break; default: ace->ae_id = cpu_to_be32(ACL_UNDEFINED_ID); @@ -120,102 +126,86 @@ xfs_acl_to_disk(struct xfs_acl *aclp, const struct posix_acl *acl) struct posix_acl * xfs_get_acl(struct inode *inode, int type) { - struct xfs_inode *ip = XFS_I(inode); - struct posix_acl *acl = NULL; - struct xfs_acl *xfs_acl = NULL; - unsigned char *ea_name; - int error; - int len; + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + struct posix_acl *acl = NULL; + struct xfs_da_args args = { + .dp = ip, + .attr_filter = XFS_ATTR_ROOT, + .valuelen = XFS_ACL_MAX_SIZE(mp), + }; + int error; trace_xfs_get_acl(ip); switch (type) { case ACL_TYPE_ACCESS: - ea_name = SGI_ACL_FILE; + args.name = SGI_ACL_FILE; break; case ACL_TYPE_DEFAULT: - ea_name = SGI_ACL_DEFAULT; + args.name = SGI_ACL_DEFAULT; break; default: BUG(); } + args.namelen = strlen(args.name); /* - * If we have a cached ACLs value just return it, not need to - * go out to the disk. + * If the attribute doesn't exist make sure we have a negative cache + * entry, for any other error assume it is transient. */ - len = XFS_ACL_MAX_SIZE(ip->i_mount); - error = xfs_attr_get(ip, ea_name, strlen(ea_name), - (unsigned char **)&xfs_acl, &len, - ATTR_ALLOC | ATTR_ROOT); - if (error) { - /* - * If the attribute doesn't exist make sure we have a negative - * cache entry, for any other error assume it is transient. - */ - if (error != -ENOATTR) - acl = ERR_PTR(error); - } else { - acl = xfs_acl_from_disk(ip->i_mount, xfs_acl, len, - XFS_ACL_MAX_ENTRIES(ip->i_mount)); - kmem_free(xfs_acl); + error = xfs_attr_get(&args); + if (!error) { + acl = xfs_acl_from_disk(mp, args.value, args.valuelen, + XFS_ACL_MAX_ENTRIES(mp)); + } else if (error != -ENOATTR) { + acl = ERR_PTR(error); } + + kmem_free(args.value); return acl; } int __xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) { - struct xfs_inode *ip = XFS_I(inode); - unsigned char *ea_name; - int error; + struct xfs_inode *ip = XFS_I(inode); + struct xfs_da_args args = { + .dp = ip, + .attr_filter = XFS_ATTR_ROOT, + }; + int error; switch (type) { case ACL_TYPE_ACCESS: - ea_name = SGI_ACL_FILE; + args.name = SGI_ACL_FILE; break; case ACL_TYPE_DEFAULT: if (!S_ISDIR(inode->i_mode)) return acl ? -EACCES : 0; - ea_name = SGI_ACL_DEFAULT; + args.name = SGI_ACL_DEFAULT; break; default: return -EINVAL; } + args.namelen = strlen(args.name); if (acl) { - struct xfs_acl *xfs_acl; - int len = XFS_ACL_MAX_SIZE(ip->i_mount); - - xfs_acl = kmem_zalloc_large(len, 0); - if (!xfs_acl) + args.valuelen = XFS_ACL_SIZE(acl->a_count); + args.value = kmem_zalloc_large(args.valuelen, 0); + if (!args.value) return -ENOMEM; - - xfs_acl_to_disk(xfs_acl, acl); - - /* subtract away the unused acl entries */ - len -= sizeof(struct xfs_acl_entry) * - (XFS_ACL_MAX_ENTRIES(ip->i_mount) - acl->a_count); - - error = xfs_attr_set(ip, ea_name, strlen(ea_name), - (unsigned char *)xfs_acl, len, ATTR_ROOT); - - kmem_free(xfs_acl); - } else { - /* - * A NULL ACL argument means we want to remove the ACL. - */ - error = xfs_attr_remove(ip, ea_name, - strlen(ea_name), - ATTR_ROOT); - - /* - * If the attribute didn't exist to start with that's fine. - */ - if (error == -ENOATTR) - error = 0; + xfs_acl_to_disk(args.value, acl); } + error = xfs_attr_set(&args); + kmem_free(args.value); + + /* + * If the attribute didn't exist to start with that's fine. + */ + if (!acl && error == -ENOATTR) + error = 0; if (!error) set_cached_acl(inode, type, acl); return error; @@ -275,3 +265,19 @@ xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) return error; } + +/* + * Invalidate any cached ACLs if the user has bypassed the ACL interface. + * We don't validate the content whatsoever so it is caller responsibility to + * provide data in valid format and ensure i_mode is consistent. + */ +void +xfs_forget_acl( + struct inode *inode, + const char *name) +{ + if (!strcmp(name, SGI_ACL_FILE)) + forget_cached_acl(inode, ACL_TYPE_ACCESS); + else if (!strcmp(name, SGI_ACL_DEFAULT)) + forget_cached_acl(inode, ACL_TYPE_DEFAULT); +} diff --git a/fs/xfs/xfs_acl.h b/fs/xfs/xfs_acl.h index 94615e34bc86..c042c0868016 100644 --- a/fs/xfs/xfs_acl.h +++ b/fs/xfs/xfs_acl.h @@ -13,14 +13,16 @@ struct posix_acl; extern struct posix_acl *xfs_get_acl(struct inode *inode, int type); extern int xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type); extern int __xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type); +void xfs_forget_acl(struct inode *inode, const char *name); #else static inline struct posix_acl *xfs_get_acl(struct inode *inode, int type) { return NULL; } # define xfs_set_acl NULL +static inline void xfs_forget_acl(struct inode *inode, const char *name) +{ +} #endif /* CONFIG_XFS_POSIX_ACL */ -extern void xfs_forget_acl(struct inode *inode, const char *name, int xflags); - #endif /* __XFS_ACL_H__ */ diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 58e937be24ce..9d9cebf18726 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -539,7 +539,7 @@ xfs_discard_page( if (XFS_FORCED_SHUTDOWN(mp)) goto out_invalidate; - xfs_alert(mp, + xfs_alert_ratelimited(mp, "page discard on page "PTR_FMT", inode 0x%llx, offset %llu.", page, ip->i_ino, offset); diff --git a/fs/xfs/xfs_attr_inactive.c b/fs/xfs/xfs_attr_inactive.c index bbfa6ba84dcd..c42f90e16b4f 100644 --- a/fs/xfs/xfs_attr_inactive.c +++ b/fs/xfs/xfs_attr_inactive.c @@ -145,8 +145,8 @@ xfs_attr3_node_inactive( * Since this code is recursive (gasp!) we must protect ourselves. */ if (level > XFS_DA_NODE_MAXDEPTH) { + xfs_buf_mark_corrupt(bp); xfs_trans_brelse(*trans, bp); /* no locks for later trans */ - xfs_buf_corruption_error(bp); return -EFSCORRUPTED; } @@ -194,7 +194,7 @@ xfs_attr3_node_inactive( error = xfs_attr3_leaf_inactive(trans, dp, child_bp); break; default: - xfs_buf_corruption_error(child_bp); + xfs_buf_mark_corrupt(child_bp); xfs_trans_brelse(*trans, child_bp); error = -EFSCORRUPTED; break; @@ -289,7 +289,7 @@ xfs_attr3_root_inactive( break; default: error = -EFSCORRUPTED; - xfs_buf_corruption_error(bp); + xfs_buf_mark_corrupt(bp); xfs_trans_brelse(*trans, bp); break; } diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index d37743bdf274..5ff1d929d3b5 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -52,24 +52,19 @@ static int xfs_attr_shortform_list( struct xfs_attr_list_context *context) { - struct attrlist_cursor_kern *cursor; + struct xfs_attrlist_cursor_kern *cursor = &context->cursor; + struct xfs_inode *dp = context->dp; struct xfs_attr_sf_sort *sbuf, *sbp; struct xfs_attr_shortform *sf; struct xfs_attr_sf_entry *sfe; - struct xfs_inode *dp; int sbsize, nsbuf, count, i; int error = 0; - ASSERT(context != NULL); - dp = context->dp; - ASSERT(dp != NULL); ASSERT(dp->i_afp != NULL); sf = (xfs_attr_shortform_t *)dp->i_afp->if_u1.if_data; ASSERT(sf != NULL); if (!sf->hdr.count) return 0; - cursor = context->cursor; - ASSERT(cursor != NULL); trace_xfs_attr_list_sf(context); @@ -205,7 +200,7 @@ xfs_attr_shortform_list( STATIC int xfs_attr_node_list_lookup( struct xfs_attr_list_context *context, - struct attrlist_cursor_kern *cursor, + struct xfs_attrlist_cursor_kern *cursor, struct xfs_buf **pbp) { struct xfs_da3_icnode_hdr nodehdr; @@ -279,7 +274,7 @@ xfs_attr_node_list_lookup( return 0; out_corruptbuf: - xfs_buf_corruption_error(bp); + xfs_buf_mark_corrupt(bp); xfs_trans_brelse(tp, bp); return -EFSCORRUPTED; } @@ -288,8 +283,8 @@ STATIC int xfs_attr_node_list( struct xfs_attr_list_context *context) { + struct xfs_attrlist_cursor_kern *cursor = &context->cursor; struct xfs_attr3_icleaf_hdr leafhdr; - struct attrlist_cursor_kern *cursor; struct xfs_attr_leafblock *leaf; struct xfs_da_intnode *node; struct xfs_buf *bp; @@ -299,7 +294,6 @@ xfs_attr_node_list( trace_xfs_attr_node_list(context); - cursor = context->cursor; cursor->initted = 1; /* @@ -394,7 +388,7 @@ xfs_attr3_leaf_list_int( struct xfs_buf *bp, struct xfs_attr_list_context *context) { - struct attrlist_cursor_kern *cursor; + struct xfs_attrlist_cursor_kern *cursor = &context->cursor; struct xfs_attr_leafblock *leaf; struct xfs_attr3_icleaf_hdr ichdr; struct xfs_attr_leaf_entry *entries; @@ -408,7 +402,6 @@ xfs_attr3_leaf_list_int( xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, leaf); entries = xfs_attr3_leaf_entryp(leaf); - cursor = context->cursor; cursor->initted = 1; /* @@ -452,8 +445,8 @@ xfs_attr3_leaf_list_int( } if ((entry->flags & XFS_ATTR_INCOMPLETE) && - !(context->flags & ATTR_INCOMPLETE)) - continue; /* skip incomplete entries */ + !context->allow_incomplete) + continue; if (entry->flags & XFS_ATTR_LOCAL) { xfs_attr_leaf_name_local_t *name_loc; @@ -488,14 +481,15 @@ xfs_attr3_leaf_list_int( * Copy out attribute entries for attr_list(), for leaf attribute lists. */ STATIC int -xfs_attr_leaf_list(xfs_attr_list_context_t *context) +xfs_attr_leaf_list( + struct xfs_attr_list_context *context) { - int error; - struct xfs_buf *bp; + struct xfs_buf *bp; + int error; trace_xfs_attr_leaf_list(context); - context->cursor->blkno = 0; + context->cursor.blkno = 0; error = xfs_attr3_leaf_read(context->tp, context->dp, 0, &bp); if (error) return error; @@ -506,7 +500,7 @@ xfs_attr_leaf_list(xfs_attr_list_context_t *context) } int -xfs_attr_list_int_ilocked( +xfs_attr_list_ilocked( struct xfs_attr_list_context *context) { struct xfs_inode *dp = context->dp; @@ -526,12 +520,12 @@ xfs_attr_list_int_ilocked( } int -xfs_attr_list_int( - xfs_attr_list_context_t *context) +xfs_attr_list( + struct xfs_attr_list_context *context) { - int error; - xfs_inode_t *dp = context->dp; - uint lock_mode; + struct xfs_inode *dp = context->dp; + uint lock_mode; + int error; XFS_STATS_INC(dp->i_mount, xs_attr_list); @@ -539,130 +533,7 @@ xfs_attr_list_int( return -EIO; lock_mode = xfs_ilock_attr_map_shared(dp); - error = xfs_attr_list_int_ilocked(context); + error = xfs_attr_list_ilocked(context); xfs_iunlock(dp, lock_mode); return error; } - -#define ATTR_ENTBASESIZE /* minimum bytes used by an attr */ \ - (((struct attrlist_ent *) 0)->a_name - (char *) 0) -#define ATTR_ENTSIZE(namelen) /* actual bytes used by an attr */ \ - ((ATTR_ENTBASESIZE + (namelen) + 1 + sizeof(uint32_t)-1) \ - & ~(sizeof(uint32_t)-1)) - -/* - * Format an attribute and copy it out to the user's buffer. - * Take care to check values and protect against them changing later, - * we may be reading them directly out of a user buffer. - */ -STATIC void -xfs_attr_put_listent( - xfs_attr_list_context_t *context, - int flags, - unsigned char *name, - int namelen, - int valuelen) -{ - struct attrlist *alist = (struct attrlist *)context->alist; - attrlist_ent_t *aep; - int arraytop; - - ASSERT(!context->seen_enough); - ASSERT(!(context->flags & ATTR_KERNOVAL)); - ASSERT(context->count >= 0); - ASSERT(context->count < (ATTR_MAX_VALUELEN/8)); - ASSERT(context->firstu >= sizeof(*alist)); - ASSERT(context->firstu <= context->bufsize); - - /* - * Only list entries in the right namespace. - */ - if (((context->flags & ATTR_SECURE) == 0) != - ((flags & XFS_ATTR_SECURE) == 0)) - return; - if (((context->flags & ATTR_ROOT) == 0) != - ((flags & XFS_ATTR_ROOT) == 0)) - return; - - arraytop = sizeof(*alist) + - context->count * sizeof(alist->al_offset[0]); - context->firstu -= ATTR_ENTSIZE(namelen); - if (context->firstu < arraytop) { - trace_xfs_attr_list_full(context); - alist->al_more = 1; - context->seen_enough = 1; - return; - } - - aep = (attrlist_ent_t *)&context->alist[context->firstu]; - aep->a_valuelen = valuelen; - memcpy(aep->a_name, name, namelen); - aep->a_name[namelen] = 0; - alist->al_offset[context->count++] = context->firstu; - alist->al_count = context->count; - trace_xfs_attr_list_add(context); - return; -} - -/* - * Generate a list of extended attribute names and optionally - * also value lengths. Positive return value follows the XFS - * convention of being an error, zero or negative return code - * is the length of the buffer returned (negated), indicating - * success. - */ -int -xfs_attr_list( - xfs_inode_t *dp, - char *buffer, - int bufsize, - int flags, - attrlist_cursor_kern_t *cursor) -{ - xfs_attr_list_context_t context; - struct attrlist *alist; - int error; - - /* - * Validate the cursor. - */ - if (cursor->pad1 || cursor->pad2) - return -EINVAL; - if ((cursor->initted == 0) && - (cursor->hashval || cursor->blkno || cursor->offset)) - return -EINVAL; - - /* Only internal consumers can retrieve incomplete attrs. */ - if (flags & ATTR_INCOMPLETE) - return -EINVAL; - - /* - * Check for a properly aligned buffer. - */ - if (((long)buffer) & (sizeof(int)-1)) - return -EFAULT; - if (flags & ATTR_KERNOVAL) - bufsize = 0; - - /* - * Initialize the output buffer. - */ - memset(&context, 0, sizeof(context)); - context.dp = dp; - context.cursor = cursor; - context.resynch = 1; - context.flags = flags; - context.alist = buffer; - context.bufsize = (bufsize & ~(sizeof(int)-1)); /* align */ - context.firstu = context.bufsize; - context.put_listent = xfs_attr_put_listent; - - alist = (struct attrlist *)context.alist; - alist->al_count = 0; - alist->al_more = 0; - alist->al_offset[0] = context.bufsize; - - error = xfs_attr_list_int(&context); - ASSERT(error <= 0); - return error; -} diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index e62fb5216341..4f800f7fe888 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -1062,7 +1062,6 @@ xfs_collapse_file_space( int error; xfs_fileoff_t next_fsb = XFS_B_TO_FSB(mp, offset + len); xfs_fileoff_t shift_fsb = XFS_B_TO_FSB(mp, len); - uint resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0); bool done = false; ASSERT(xfs_isilocked(ip, XFS_IOLOCK_EXCL)); @@ -1078,32 +1077,34 @@ xfs_collapse_file_space( if (error) return error; - while (!error && !done) { - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, - &tp); - if (error) - break; + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); + if (error) + return error; - xfs_ilock(ip, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota(tp, mp, ip->i_udquot, - ip->i_gdquot, ip->i_pdquot, resblks, 0, - XFS_QMOPT_RES_REGBLKS); - if (error) - goto out_trans_cancel; - xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + while (!done) { error = xfs_bmap_collapse_extents(tp, ip, &next_fsb, shift_fsb, &done); if (error) goto out_trans_cancel; + if (done) + break; - error = xfs_trans_commit(tp); + /* finish any deferred frees and roll the transaction */ + error = xfs_defer_finish(&tp); + if (error) + goto out_trans_cancel; } + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); return error; out_trans_cancel: xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); return error; } @@ -1146,35 +1147,41 @@ xfs_insert_file_space( if (error) return error; + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, + XFS_DIOSTRAT_SPACE_RES(mp, 0), 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + /* * The extent shifting code works on extent granularity. So, if stop_fsb * is not the starting block of extent, we need to split the extent at * stop_fsb. */ - error = xfs_bmap_split_extent(ip, stop_fsb); + error = xfs_bmap_split_extent(tp, ip, stop_fsb); if (error) - return error; + goto out_trans_cancel; - while (!error && !done) { - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, - &tp); + do { + error = xfs_trans_roll_inode(&tp, ip); if (error) - break; + goto out_trans_cancel; - xfs_ilock(ip, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); error = xfs_bmap_insert_extents(tp, ip, &next_fsb, shift_fsb, &done, stop_fsb); if (error) goto out_trans_cancel; + } while (!done); - error = xfs_trans_commit(tp); - } - + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); return error; out_trans_cancel: xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); return error; } @@ -1442,12 +1449,12 @@ xfs_swap_extent_forks( * event of a crash. Set the owner change log flags now and leave the * bmbt scan as the last step. */ - if (ip->i_d.di_version == 3 && - ip->i_d.di_format == XFS_DINODE_FMT_BTREE) - (*target_log_flags) |= XFS_ILOG_DOWNER; - if (tip->i_d.di_version == 3 && - tip->i_d.di_format == XFS_DINODE_FMT_BTREE) - (*src_log_flags) |= XFS_ILOG_DOWNER; + if (xfs_sb_version_has_v3inode(&ip->i_mount->m_sb)) { + if (ip->i_d.di_format == XFS_DINODE_FMT_BTREE) + (*target_log_flags) |= XFS_ILOG_DOWNER; + if (tip->i_d.di_format == XFS_DINODE_FMT_BTREE) + (*src_log_flags) |= XFS_ILOG_DOWNER; + } /* * Swap the data forks of the inodes @@ -1482,7 +1489,7 @@ xfs_swap_extent_forks( (*src_log_flags) |= XFS_ILOG_DEXT; break; case XFS_DINODE_FMT_BTREE: - ASSERT(ip->i_d.di_version < 3 || + ASSERT(!xfs_sb_version_has_v3inode(&ip->i_mount->m_sb) || (*src_log_flags & XFS_ILOG_DOWNER)); (*src_log_flags) |= XFS_ILOG_DBROOT; break; @@ -1494,7 +1501,7 @@ xfs_swap_extent_forks( break; case XFS_DINODE_FMT_BTREE: (*target_log_flags) |= XFS_ILOG_DBROOT; - ASSERT(tip->i_d.di_version < 3 || + ASSERT(!xfs_sb_version_has_v3inode(&ip->i_mount->m_sb) || (*target_log_flags & XFS_ILOG_DOWNER)); break; } diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 217e4f82a44a..f880141a2268 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -727,8 +727,9 @@ xfs_buf_get_map( if (!bp->b_addr) { error = _xfs_buf_map_pages(bp, flags); if (unlikely(error)) { - xfs_warn(target->bt_mount, - "%s: failed to map pagesn", __func__); + xfs_warn_ratelimited(target->bt_mount, + "%s: failed to map %u pages", __func__, + bp->b_page_count); xfs_buf_relse(bp); return error; } @@ -1238,7 +1239,7 @@ xfs_buf_ioerror_alert( struct xfs_buf *bp, xfs_failaddr_t func) { - xfs_alert(bp->b_mount, + xfs_alert_ratelimited(bp->b_mount, "metadata I/O error in \"%pS\" at daddr 0x%llx len %d error %d", func, (uint64_t)XFS_BUF_ADDR(bp), bp->b_length, -bp->b_error); @@ -1572,6 +1573,28 @@ xfs_buf_zero( } } +/* + * Log a message about and stale a buffer that a caller has decided is corrupt. + * + * This function should be called for the kinds of metadata corruption that + * cannot be detect from a verifier, such as incorrect inter-block relationship + * data. Do /not/ call this function from a verifier function. + * + * The buffer must be XBF_DONE prior to the call. Afterwards, the buffer will + * be marked stale, but b_error will not be set. The caller is responsible for + * releasing the buffer or fixing it. + */ +void +__xfs_buf_mark_corrupt( + struct xfs_buf *bp, + xfs_failaddr_t fa) +{ + ASSERT(bp->b_flags & XBF_DONE); + + xfs_buf_corruption_error(bp, fa); + xfs_buf_stale(bp); +} + /* * Handling of buffer targets (buftargs). */ diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index d79a1fe5d738..9a04c53c2488 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -272,6 +272,8 @@ static inline int xfs_buf_submit(struct xfs_buf *bp) } void xfs_buf_zero(struct xfs_buf *bp, size_t boff, size_t bsize); +void __xfs_buf_mark_corrupt(struct xfs_buf *bp, xfs_failaddr_t fa); +#define xfs_buf_mark_corrupt(bp) __xfs_buf_mark_corrupt((bp), __this_address) /* Buffer Utility Routines */ extern void *xfs_buf_offset(struct xfs_buf *, size_t); diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 663810e6cd59..1545657c3ca0 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -345,7 +345,7 @@ xfs_buf_item_format( * occurs during recovery. */ if (bip->bli_flags & XFS_BLI_INODE_BUF) { - if (xfs_sb_version_hascrc(&lip->li_mountp->m_sb) || + if (xfs_sb_version_has_v3inode(&lip->li_mountp->m_sb) || !((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && xfs_log_item_in_current_chkpt(lip))) bip->__bli_format.blf_flags |= XFS_BLF_INODE_BUF; diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c index 0d3b640cf1cc..871ec22c9aee 100644 --- a/fs/xfs/xfs_dir2_readdir.c +++ b/fs/xfs/xfs_dir2_readdir.c @@ -147,7 +147,7 @@ xfs_dir2_block_getdents( xfs_off_t cook; struct xfs_da_geometry *geo = args->geo; int lock_mode; - unsigned int offset; + unsigned int offset, next_offset; unsigned int end; /* @@ -173,9 +173,10 @@ xfs_dir2_block_getdents( * Loop over the data portion of the block. * Each object is a real entry (dep) or an unused one (dup). */ - offset = geo->data_entry_offset; end = xfs_dir3_data_end_offset(geo, bp->b_addr); - while (offset < end) { + for (offset = geo->data_entry_offset; + offset < end; + offset = next_offset) { struct xfs_dir2_data_unused *dup = bp->b_addr + offset; struct xfs_dir2_data_entry *dep = bp->b_addr + offset; uint8_t filetype; @@ -184,14 +185,15 @@ xfs_dir2_block_getdents( * Unused, skip it. */ if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { - offset += be16_to_cpu(dup->length); + next_offset = offset + be16_to_cpu(dup->length); continue; } /* * Bump pointer for the next iteration. */ - offset += xfs_dir2_data_entsize(dp->i_mount, dep->namelen); + next_offset = offset + + xfs_dir2_data_entsize(dp->i_mount, dep->namelen); /* * The entry is before the desired starting point, skip it. diff --git a/fs/xfs/xfs_discard.c b/fs/xfs/xfs_discard.c index 0b8350e84d28..f979d0d7e6cd 100644 --- a/fs/xfs/xfs_discard.c +++ b/fs/xfs/xfs_discard.c @@ -31,6 +31,7 @@ xfs_trim_extents( struct block_device *bdev = mp->m_ddev_targp->bt_bdev; struct xfs_btree_cur *cur; struct xfs_buf *agbp; + struct xfs_agf *agf; struct xfs_perag *pag; int error; int i; @@ -47,14 +48,14 @@ xfs_trim_extents( error = xfs_alloc_read_agf(mp, NULL, agno, 0, &agbp); if (error) goto out_put_perag; + agf = agbp->b_addr; cur = xfs_allocbt_init_cursor(mp, NULL, agbp, agno, XFS_BTNUM_CNT); /* * Look up the longest btree in the AGF and start with it. */ - error = xfs_alloc_lookup_ge(cur, 0, - be32_to_cpu(XFS_BUF_TO_AGF(agbp)->agf_longest), &i); + error = xfs_alloc_lookup_ge(cur, 0, be32_to_cpu(agf->agf_longest), &i); if (error) goto out_del_cursor; @@ -75,7 +76,7 @@ xfs_trim_extents( error = -EFSCORRUPTED; goto out_del_cursor; } - ASSERT(flen <= be32_to_cpu(XFS_BUF_TO_AGF(agbp)->agf_longest)); + ASSERT(flen <= be32_to_cpu(agf->agf_longest)); /* * use daddr format for all range/len calculations as that is diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index d223e1ae90a6..711376ca269f 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -829,9 +829,9 @@ xfs_qm_id_for_quotatype( { switch (type) { case XFS_DQ_USER: - return ip->i_d.di_uid; + return i_uid_read(VFS_I(ip)); case XFS_DQ_GROUP: - return ip->i_d.di_gid; + return i_gid_read(VFS_I(ip)); case XFS_DQ_PROJ: return ip->i_d.di_projid; } diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c index d60647d7197b..cf65e2e43c6e 100644 --- a/fs/xfs/xfs_dquot_item.c +++ b/fs/xfs/xfs_dquot_item.c @@ -307,35 +307,61 @@ xfs_qm_qoffend_logitem_committed( { struct xfs_qoff_logitem *qfe = QOFF_ITEM(lip); struct xfs_qoff_logitem *qfs = qfe->qql_start_lip; - struct xfs_ail *ailp = qfs->qql_item.li_ailp; - /* - * Delete the qoff-start logitem from the AIL. - * xfs_trans_ail_delete() drops the AIL lock. - */ - spin_lock(&ailp->ail_lock); - xfs_trans_ail_delete(ailp, &qfs->qql_item, SHUTDOWN_LOG_IO_ERROR); + xfs_qm_qoff_logitem_relse(qfs); - kmem_free(qfs->qql_item.li_lv_shadow); kmem_free(lip->li_lv_shadow); - kmem_free(qfs); kmem_free(qfe); return (xfs_lsn_t)-1; } +STATIC void +xfs_qm_qoff_logitem_release( + struct xfs_log_item *lip) +{ + struct xfs_qoff_logitem *qoff = QOFF_ITEM(lip); + + if (test_bit(XFS_LI_ABORTED, &lip->li_flags)) { + if (qoff->qql_start_lip) + xfs_qm_qoff_logitem_relse(qoff->qql_start_lip); + xfs_qm_qoff_logitem_relse(qoff); + } +} + static const struct xfs_item_ops xfs_qm_qoffend_logitem_ops = { .iop_size = xfs_qm_qoff_logitem_size, .iop_format = xfs_qm_qoff_logitem_format, .iop_committed = xfs_qm_qoffend_logitem_committed, .iop_push = xfs_qm_qoff_logitem_push, + .iop_release = xfs_qm_qoff_logitem_release, }; static const struct xfs_item_ops xfs_qm_qoff_logitem_ops = { .iop_size = xfs_qm_qoff_logitem_size, .iop_format = xfs_qm_qoff_logitem_format, .iop_push = xfs_qm_qoff_logitem_push, + .iop_release = xfs_qm_qoff_logitem_release, }; +/* + * Delete the quotaoff intent from the AIL and free it. On success, + * this should only be called for the start item. It can be used for + * either on shutdown or abort. + */ +void +xfs_qm_qoff_logitem_relse( + struct xfs_qoff_logitem *qoff) +{ + struct xfs_log_item *lip = &qoff->qql_item; + + ASSERT(test_bit(XFS_LI_IN_AIL, &lip->li_flags) || + test_bit(XFS_LI_ABORTED, &lip->li_flags) || + XFS_FORCED_SHUTDOWN(lip->li_mountp)); + xfs_trans_ail_remove(lip, SHUTDOWN_LOG_IO_ERROR); + kmem_free(lip->li_lv_shadow); + kmem_free(qoff); +} + /* * Allocate and initialize an quotaoff item of the correct quota type(s). */ diff --git a/fs/xfs/xfs_dquot_item.h b/fs/xfs/xfs_dquot_item.h index 3bb19e556ade..2b86a43d7ce2 100644 --- a/fs/xfs/xfs_dquot_item.h +++ b/fs/xfs/xfs_dquot_item.h @@ -28,6 +28,7 @@ void xfs_qm_dquot_logitem_init(struct xfs_dquot *dqp); struct xfs_qoff_logitem *xfs_qm_qoff_logitem_init(struct xfs_mount *mp, struct xfs_qoff_logitem *start, uint flags); +void xfs_qm_qoff_logitem_relse(struct xfs_qoff_logitem *); struct xfs_qoff_logitem *xfs_trans_get_qoff_item(struct xfs_trans *tp, struct xfs_qoff_logitem *startqoff, uint flags); diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c index 331765afc53e..a21e9cc6516a 100644 --- a/fs/xfs/xfs_error.c +++ b/fs/xfs/xfs_error.c @@ -345,16 +345,19 @@ xfs_corruption_error( * Complain about the kinds of metadata corruption that we can't detect from a * verifier, such as incorrect inter-block relationship data. Does not set * bp->b_error. + * + * Call xfs_buf_mark_corrupt, not this function. */ void xfs_buf_corruption_error( - struct xfs_buf *bp) + struct xfs_buf *bp, + xfs_failaddr_t fa) { struct xfs_mount *mp = bp->b_mount; xfs_alert_tag(mp, XFS_PTAG_VERIFIER_ERROR, "Metadata corruption detected at %pS, %s block 0x%llx", - __return_address, bp->b_ops->name, bp->b_bn); + fa, bp->b_ops->name, bp->b_bn); xfs_alert(mp, "Unmount and run xfs_repair"); diff --git a/fs/xfs/xfs_error.h b/fs/xfs/xfs_error.h index 31a5d321ba9a..1717b7508356 100644 --- a/fs/xfs/xfs_error.h +++ b/fs/xfs/xfs_error.h @@ -15,7 +15,7 @@ extern void xfs_corruption_error(const char *tag, int level, struct xfs_mount *mp, const void *buf, size_t bufsize, const char *filename, int linenum, xfs_failaddr_t failaddr); -void xfs_buf_corruption_error(struct xfs_buf *bp); +void xfs_buf_corruption_error(struct xfs_buf *bp, xfs_failaddr_t fa); extern void xfs_buf_verifier_error(struct xfs_buf *bp, int error, const char *name, const void *buf, size_t bufsz, xfs_failaddr_t failaddr); diff --git a/fs/xfs/xfs_fsmap.c b/fs/xfs/xfs_fsmap.c index 918456ca29e1..4eebcec4aae6 100644 --- a/fs/xfs/xfs_fsmap.c +++ b/fs/xfs/xfs_fsmap.c @@ -344,7 +344,7 @@ xfs_getfsmap_datadev_helper( xfs_fsblock_t fsb; xfs_daddr_t rec_daddr; - fsb = XFS_AGB_TO_FSB(mp, cur->bc_private.a.agno, rec->rm_startblock); + fsb = XFS_AGB_TO_FSB(mp, cur->bc_ag.agno, rec->rm_startblock); rec_daddr = XFS_FSB_TO_DADDR(mp, fsb); return xfs_getfsmap_helper(cur->bc_tp, info, rec, rec_daddr); @@ -362,7 +362,7 @@ xfs_getfsmap_datadev_bnobt_helper( struct xfs_rmap_irec irec; xfs_daddr_t rec_daddr; - rec_daddr = XFS_AGB_TO_DADDR(mp, cur->bc_private.a.agno, + rec_daddr = XFS_AGB_TO_DADDR(mp, cur->bc_ag.agno, rec->ar_startblock); irec.rm_startblock = rec->ar_startblock; @@ -896,6 +896,14 @@ xfs_getfsmap( info.format_arg = arg; info.head = head; + /* + * If fsmap runs concurrently with a scrub, the freeze can be delayed + * indefinitely as we walk the rmapbt and iterate over metadata + * buffers. Freeze quiesces the log (which waits for the buffer LRU to + * be emptied) and that won't happen while we're reading buffers. + */ + sb_start_write(mp->m_super); + /* For each device we support... */ for (i = 0; i < XFS_GETFSMAP_DEVS; i++) { /* Is this device within the range the user asked for? */ @@ -935,6 +943,7 @@ xfs_getfsmap( if (tp) xfs_trans_cancel(tp); + sb_end_write(mp->m_super); head->fmh_oflags = FMH_OF_DEV_T; return error; } diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 8dc2e5414276..a7be7a9e5c1a 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -289,6 +289,8 @@ xfs_reinit_inode( uint64_t version = inode_peek_iversion(inode); umode_t mode = inode->i_mode; dev_t dev = inode->i_rdev; + kuid_t uid = inode->i_uid; + kgid_t gid = inode->i_gid; error = inode_init_always(mp->m_super, inode); @@ -297,6 +299,8 @@ xfs_reinit_inode( inode_set_iversion_queried(inode, version); inode->i_mode = mode; inode->i_rdev = dev; + inode->i_uid = uid; + inode->i_gid = gid; return error; } diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index c5077e6326c7..14b922f2a6db 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -801,26 +801,18 @@ xfs_ialloc( return error; ASSERT(ip != NULL); inode = VFS_I(ip); - - /* - * We always convert v1 inodes to v2 now - we only support filesystems - * with >= v2 inode capability, so there is no reason for ever leaving - * an inode in v1 format. - */ - if (ip->i_d.di_version == 1) - ip->i_d.di_version = 2; - inode->i_mode = mode; set_nlink(inode, nlink); - ip->i_d.di_uid = xfs_kuid_to_uid(current_fsuid()); - ip->i_d.di_gid = xfs_kgid_to_gid(current_fsgid()); + inode->i_uid = current_fsuid(); inode->i_rdev = rdev; ip->i_d.di_projid = prid; if (pip && XFS_INHERIT_GID(pip)) { - ip->i_d.di_gid = pip->i_d.di_gid; + inode->i_gid = VFS_I(pip)->i_gid; if ((VFS_I(pip)->i_mode & S_ISGID) && S_ISDIR(mode)) inode->i_mode |= S_ISGID; + } else { + inode->i_gid = current_fsgid(); } /* @@ -828,9 +820,8 @@ xfs_ialloc( * ID or one of the supplementary group IDs, the S_ISGID bit is cleared * (and only if the irix_sgid_inherit compatibility variable is set). */ - if ((irix_sgid_inherit) && - (inode->i_mode & S_ISGID) && - (!in_group_p(xfs_gid_to_kgid(ip->i_d.di_gid)))) + if (irix_sgid_inherit && + (inode->i_mode & S_ISGID) && !in_group_p(inode->i_gid)) inode->i_mode &= ~S_ISGID; ip->i_d.di_size = 0; @@ -847,14 +838,13 @@ xfs_ialloc( ip->i_d.di_dmstate = 0; ip->i_d.di_flags = 0; - if (ip->i_d.di_version == 3) { + if (xfs_sb_version_has_v3inode(&mp->m_sb)) { inode_set_iversion(inode, 1); ip->i_d.di_flags2 = 0; ip->i_d.di_cowextsize = 0; ip->i_d.di_crtime = tv; } - flags = XFS_ILOG_CORE; switch (mode & S_IFMT) { case S_IFIFO: @@ -907,20 +897,13 @@ xfs_ialloc( ip->i_d.di_flags |= di_flags; } - if (pip && - (pip->i_d.di_flags2 & XFS_DIFLAG2_ANY) && - pip->i_d.di_version == 3 && - ip->i_d.di_version == 3) { - uint64_t di_flags2 = 0; - + if (pip && (pip->i_d.di_flags2 & XFS_DIFLAG2_ANY)) { if (pip->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE) { - di_flags2 |= XFS_DIFLAG2_COWEXTSIZE; + ip->i_d.di_flags2 |= XFS_DIFLAG2_COWEXTSIZE; ip->i_d.di_cowextsize = pip->i_d.di_cowextsize; } if (pip->i_d.di_flags2 & XFS_DIFLAG2_DAX) - di_flags2 |= XFS_DIFLAG2_DAX; - - ip->i_d.di_flags2 |= di_flags2; + ip->i_d.di_flags2 |= XFS_DIFLAG2_DAX; } /* FALLTHROUGH */ case S_IFLNK: @@ -1122,7 +1105,6 @@ xfs_bumplink( { xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_CHG); - ASSERT(ip->i_d.di_version > 1); inc_nlink(VFS_I(ip)); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); } @@ -1158,8 +1140,7 @@ xfs_create( /* * Make sure that we have allocated dquot(s) on disk. */ - error = xfs_qm_vop_dqalloc(dp, xfs_kuid_to_uid(current_fsuid()), - xfs_kgid_to_gid(current_fsgid()), prid, + error = xfs_qm_vop_dqalloc(dp, current_fsuid(), current_fsgid(), prid, XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, &udqp, &gdqp, &pdqp); if (error) @@ -1309,8 +1290,7 @@ xfs_create_tmpfile( /* * Make sure that we have allocated dquot(s) on disk. */ - error = xfs_qm_vop_dqalloc(dp, xfs_kuid_to_uid(current_fsuid()), - xfs_kgid_to_gid(current_fsgid()), prid, + error = xfs_qm_vop_dqalloc(dp, current_fsuid(), current_fsgid(), prid, XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, &udqp, &gdqp, &pdqp); if (error) @@ -2119,7 +2099,7 @@ xfs_iunlink_update_bucket( unsigned int bucket_index, xfs_agino_t new_agino) { - struct xfs_agi *agi = XFS_BUF_TO_AGI(agibp); + struct xfs_agi *agi = agibp->b_addr; xfs_agino_t old_value; int offset; @@ -2135,7 +2115,7 @@ xfs_iunlink_update_bucket( * head of the list. */ if (old_value == new_agino) { - xfs_buf_corruption_error(agibp); + xfs_buf_mark_corrupt(agibp); return -EFSCORRUPTED; } @@ -2259,7 +2239,7 @@ xfs_iunlink( error = xfs_read_agi(mp, tp, agno, &agibp); if (error) return error; - agi = XFS_BUF_TO_AGI(agibp); + agi = agibp->b_addr; /* * Get the index into the agi hash table for the list this inode will @@ -2269,7 +2249,7 @@ xfs_iunlink( next_agino = be32_to_cpu(agi->agi_unlinked[bucket_index]); if (next_agino == agino || !xfs_verify_agino_or_null(mp, agno, next_agino)) { - xfs_buf_corruption_error(agibp); + xfs_buf_mark_corrupt(agibp); return -EFSCORRUPTED; } @@ -2443,7 +2423,7 @@ xfs_iunlink_remove( error = xfs_read_agi(mp, tp, agno, &agibp); if (error) return error; - agi = XFS_BUF_TO_AGI(agibp); + agi = agibp->b_addr; /* * Get the index into the agi hash table for the list this inode will @@ -3807,7 +3787,6 @@ xfs_iflush_int( ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE || ip->i_d.di_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)); ASSERT(iip != NULL && iip->ili_fields != 0); - ASSERT(ip->i_d.di_version > 1); /* set *dip = inode's place in the buffer */ dip = xfs_buf_offset(bp, ip->i_imap.im_boffset); @@ -3868,7 +3847,7 @@ xfs_iflush_int( * backwards compatibility with old kernels that predate logging all * inode changes. */ - if (ip->i_d.di_version < 3) + if (!xfs_sb_version_has_v3inode(&mp->m_sb)) ip->i_d.di_flushiter++; /* Check the inline fork data before we write out. */ diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 8bd5d0de6321..4a3d13d4a022 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -125,7 +125,7 @@ xfs_inode_item_size( *nvecs += 2; *nbytes += sizeof(struct xfs_inode_log_format) + - xfs_log_dinode_size(ip->i_d.di_version); + xfs_log_dinode_size(ip->i_mount); xfs_inode_item_data_fork_size(iip, nvecs, nbytes); if (XFS_IFORK_Q(ip)) @@ -305,11 +305,9 @@ xfs_inode_to_log_dinode( struct inode *inode = VFS_I(ip); to->di_magic = XFS_DINODE_MAGIC; - - to->di_version = from->di_version; to->di_format = from->di_format; - to->di_uid = from->di_uid; - to->di_gid = from->di_gid; + to->di_uid = i_uid_read(inode); + to->di_gid = i_gid_read(inode); to->di_projid_lo = from->di_projid & 0xffff; to->di_projid_hi = from->di_projid >> 16; @@ -339,7 +337,8 @@ xfs_inode_to_log_dinode( /* log a dummy value to ensure log structure is fully initialised */ to->di_next_unlinked = NULLAGINO; - if (from->di_version == 3) { + if (xfs_sb_version_has_v3inode(&ip->i_mount->m_sb)) { + to->di_version = 3; to->di_changecount = inode_peek_iversion(inode); to->di_crtime.t_sec = from->di_crtime.tv_sec; to->di_crtime.t_nsec = from->di_crtime.tv_nsec; @@ -351,6 +350,7 @@ xfs_inode_to_log_dinode( uuid_copy(&to->di_uuid, &ip->i_mount->m_sb.sb_meta_uuid); to->di_flushiter = 0; } else { + to->di_version = 2; to->di_flushiter = from->di_flushiter; } } @@ -370,7 +370,7 @@ xfs_inode_item_format_core( dic = xlog_prepare_iovec(lv, vecp, XLOG_REG_TYPE_ICORE); xfs_inode_to_log_dinode(ip, dic, ip->i_itemp->ili_item.li_lsn); - xlog_finish_iovec(lv, *vecp, xfs_log_dinode_size(ip->i_d.di_version)); + xlog_finish_iovec(lv, *vecp, xfs_log_dinode_size(ip->i_mount)); } /* @@ -395,8 +395,6 @@ xfs_inode_item_format( struct xfs_log_iovec *vecp = NULL; struct xfs_inode_log_format *ilf; - ASSERT(ip->i_d.di_version > 1); - ilf = xlog_prepare_iovec(lv, &vecp, XLOG_REG_TYPE_IFORMAT); ilf->ilf_type = XFS_LI_INODE; ilf->ilf_ino = ip->i_ino; diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index d42de92cb283..cdfb3cd9a25b 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -35,6 +35,8 @@ #include "xfs_health.h" #include "xfs_reflink.h" #include "xfs_ioctl.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" #include #include @@ -292,62 +294,173 @@ xfs_readlink_by_handle( return error; } -STATIC int -xfs_attrlist_by_handle( - struct file *parfilp, - void __user *arg) +/* + * Format an attribute and copy it out to the user's buffer. + * Take care to check values and protect against them changing later, + * we may be reading them directly out of a user buffer. + */ +static void +xfs_ioc_attr_put_listent( + struct xfs_attr_list_context *context, + int flags, + unsigned char *name, + int namelen, + int valuelen) { - int error = -ENOMEM; - attrlist_cursor_kern_t *cursor; - struct xfs_fsop_attrlist_handlereq __user *p = arg; - xfs_fsop_attrlist_handlereq_t al_hreq; - struct dentry *dentry; - char *kbuf; + struct xfs_attrlist *alist = context->buffer; + struct xfs_attrlist_ent *aep; + int arraytop; - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - if (copy_from_user(&al_hreq, arg, sizeof(xfs_fsop_attrlist_handlereq_t))) - return -EFAULT; - if (al_hreq.buflen < sizeof(struct attrlist) || - al_hreq.buflen > XFS_XATTR_LIST_MAX) + ASSERT(!context->seen_enough); + ASSERT(context->count >= 0); + ASSERT(context->count < (ATTR_MAX_VALUELEN/8)); + ASSERT(context->firstu >= sizeof(*alist)); + ASSERT(context->firstu <= context->bufsize); + + /* + * Only list entries in the right namespace. + */ + if (context->attr_filter != (flags & XFS_ATTR_NSP_ONDISK_MASK)) + return; + + arraytop = sizeof(*alist) + + context->count * sizeof(alist->al_offset[0]); + + /* decrement by the actual bytes used by the attr */ + context->firstu -= round_up(offsetof(struct xfs_attrlist_ent, a_name) + + namelen + 1, sizeof(uint32_t)); + if (context->firstu < arraytop) { + trace_xfs_attr_list_full(context); + alist->al_more = 1; + context->seen_enough = 1; + return; + } + + aep = context->buffer + context->firstu; + aep->a_valuelen = valuelen; + memcpy(aep->a_name, name, namelen); + aep->a_name[namelen] = 0; + alist->al_offset[context->count++] = context->firstu; + alist->al_count = context->count; + trace_xfs_attr_list_add(context); +} + +static unsigned int +xfs_attr_filter( + u32 ioc_flags) +{ + if (ioc_flags & XFS_IOC_ATTR_ROOT) + return XFS_ATTR_ROOT; + if (ioc_flags & XFS_IOC_ATTR_SECURE) + return XFS_ATTR_SECURE; + return 0; +} + +static unsigned int +xfs_attr_flags( + u32 ioc_flags) +{ + if (ioc_flags & XFS_IOC_ATTR_CREATE) + return XATTR_CREATE; + if (ioc_flags & XFS_IOC_ATTR_REPLACE) + return XATTR_REPLACE; + return 0; +} + +int +xfs_ioc_attr_list( + struct xfs_inode *dp, + void __user *ubuf, + int bufsize, + int flags, + struct xfs_attrlist_cursor __user *ucursor) +{ + struct xfs_attr_list_context context = { }; + struct xfs_attrlist *alist; + void *buffer; + int error; + + if (bufsize < sizeof(struct xfs_attrlist) || + bufsize > XFS_XATTR_LIST_MAX) return -EINVAL; /* * Reject flags, only allow namespaces. */ - if (al_hreq.flags & ~(ATTR_ROOT | ATTR_SECURE)) + if (flags & ~(XFS_IOC_ATTR_ROOT | XFS_IOC_ATTR_SECURE)) return -EINVAL; + if (flags == (XFS_IOC_ATTR_ROOT | XFS_IOC_ATTR_SECURE)) + return -EINVAL; + + /* + * Validate the cursor. + */ + if (copy_from_user(&context.cursor, ucursor, sizeof(context.cursor))) + return -EFAULT; + if (context.cursor.pad1 || context.cursor.pad2) + return -EINVAL; + if (!context.cursor.initted && + (context.cursor.hashval || context.cursor.blkno || + context.cursor.offset)) + return -EINVAL; + + buffer = kmem_zalloc_large(bufsize, 0); + if (!buffer) + return -ENOMEM; + + /* + * Initialize the output buffer. + */ + context.dp = dp; + context.resynch = 1; + context.attr_filter = xfs_attr_filter(flags); + context.buffer = buffer; + context.bufsize = round_down(bufsize, sizeof(uint32_t)); + context.firstu = context.bufsize; + context.put_listent = xfs_ioc_attr_put_listent; + + alist = context.buffer; + alist->al_count = 0; + alist->al_more = 0; + alist->al_offset[0] = context.bufsize; + + error = xfs_attr_list(&context); + if (error) + goto out_free; + + if (copy_to_user(ubuf, buffer, bufsize) || + copy_to_user(ucursor, &context.cursor, sizeof(context.cursor))) + error = -EFAULT; +out_free: + kmem_free(buffer); + return error; +} + +STATIC int +xfs_attrlist_by_handle( + struct file *parfilp, + struct xfs_fsop_attrlist_handlereq __user *p) +{ + struct xfs_fsop_attrlist_handlereq al_hreq; + struct dentry *dentry; + int error = -ENOMEM; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&al_hreq, p, sizeof(al_hreq))) + return -EFAULT; dentry = xfs_handlereq_to_dentry(parfilp, &al_hreq.hreq); if (IS_ERR(dentry)) return PTR_ERR(dentry); - kbuf = kmem_zalloc_large(al_hreq.buflen, 0); - if (!kbuf) - goto out_dput; - - cursor = (attrlist_cursor_kern_t *)&al_hreq.pos; - error = xfs_attr_list(XFS_I(d_inode(dentry)), kbuf, al_hreq.buflen, - al_hreq.flags, cursor); - if (error) - goto out_kfree; - - if (copy_to_user(&p->pos, cursor, sizeof(attrlist_cursor_kern_t))) { - error = -EFAULT; - goto out_kfree; - } - - if (copy_to_user(al_hreq.buffer, kbuf, al_hreq.buflen)) - error = -EFAULT; - -out_kfree: - kmem_free(kbuf); -out_dput: + error = xfs_ioc_attr_list(XFS_I(d_inode(dentry)), al_hreq.buffer, + al_hreq.buflen, al_hreq.flags, &p->pos); dput(dentry); return error; } -int +static int xfs_attrmulti_attr_get( struct inode *inode, unsigned char *name, @@ -355,31 +468,33 @@ xfs_attrmulti_attr_get( uint32_t *len, uint32_t flags) { - unsigned char *kbuf; - int error = -EFAULT; - size_t namelen; + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = xfs_attr_filter(flags), + .attr_flags = xfs_attr_flags(flags), + .name = name, + .namelen = strlen(name), + .valuelen = *len, + }; + int error; if (*len > XFS_XATTR_SIZE_MAX) return -EINVAL; - kbuf = kmem_zalloc_large(*len, 0); - if (!kbuf) - return -ENOMEM; - namelen = strlen(name); - error = xfs_attr_get(XFS_I(inode), name, namelen, &kbuf, (int *)len, - flags); + error = xfs_attr_get(&args); if (error) goto out_kfree; - if (copy_to_user(ubuf, kbuf, *len)) + *len = args.valuelen; + if (copy_to_user(ubuf, args.value, args.valuelen)) error = -EFAULT; out_kfree: - kmem_free(kbuf); + kmem_free(args.value); return error; } -int +static int xfs_attrmulti_attr_set( struct inode *inode, unsigned char *name, @@ -387,42 +502,75 @@ xfs_attrmulti_attr_set( uint32_t len, uint32_t flags) { - unsigned char *kbuf; + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = xfs_attr_filter(flags), + .attr_flags = xfs_attr_flags(flags), + .name = name, + .namelen = strlen(name), + }; int error; - size_t namelen; if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) return -EPERM; - if (len > XFS_XATTR_SIZE_MAX) - return -EINVAL; - kbuf = memdup_user(ubuf, len); - if (IS_ERR(kbuf)) - return PTR_ERR(kbuf); + if (ubuf) { + if (len > XFS_XATTR_SIZE_MAX) + return -EINVAL; + args.value = memdup_user(ubuf, len); + if (IS_ERR(args.value)) + return PTR_ERR(args.value); + args.valuelen = len; + } - namelen = strlen(name); - error = xfs_attr_set(XFS_I(inode), name, namelen, kbuf, len, flags); - if (!error) - xfs_forget_acl(inode, name, flags); - kfree(kbuf); + error = xfs_attr_set(&args); + if (!error && (flags & XFS_IOC_ATTR_ROOT)) + xfs_forget_acl(inode, name); + kfree(args.value); return error; } int -xfs_attrmulti_attr_remove( +xfs_ioc_attrmulti_one( + struct file *parfilp, struct inode *inode, - unsigned char *name, + uint32_t opcode, + void __user *uname, + void __user *value, + uint32_t *len, uint32_t flags) { + unsigned char *name; int error; - size_t namelen; - if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) - return -EPERM; - namelen = strlen(name); - error = xfs_attr_remove(XFS_I(inode), name, namelen, flags); - if (!error) - xfs_forget_acl(inode, name, flags); + if ((flags & XFS_IOC_ATTR_ROOT) && (flags & XFS_IOC_ATTR_SECURE)) + return -EINVAL; + + name = strndup_user(uname, MAXNAMELEN); + if (IS_ERR(name)) + return PTR_ERR(name); + + switch (opcode) { + case ATTR_OP_GET: + error = xfs_attrmulti_attr_get(inode, name, value, len, flags); + break; + case ATTR_OP_REMOVE: + value = NULL; + *len = 0; + /* fall through */ + case ATTR_OP_SET: + error = mnt_want_write_file(parfilp); + if (error) + break; + error = xfs_attrmulti_attr_set(inode, name, value, *len, flags); + mnt_drop_write_file(parfilp); + break; + default: + error = -EINVAL; + break; + } + + kfree(name); return error; } @@ -436,7 +584,6 @@ xfs_attrmulti_by_handle( xfs_fsop_attrmulti_handlereq_t am_hreq; struct dentry *dentry; unsigned int i, size; - unsigned char *attr_name; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -462,63 +609,17 @@ xfs_attrmulti_by_handle( goto out_dput; } - error = -ENOMEM; - attr_name = kmalloc(MAXNAMELEN, GFP_KERNEL); - if (!attr_name) - goto out_kfree_ops; - error = 0; for (i = 0; i < am_hreq.opcount; i++) { - if ((ops[i].am_flags & ATTR_ROOT) && - (ops[i].am_flags & ATTR_SECURE)) { - ops[i].am_error = -EINVAL; - continue; - } - ops[i].am_flags &= ~ATTR_KERNEL_FLAGS; - - ops[i].am_error = strncpy_from_user((char *)attr_name, - ops[i].am_attrname, MAXNAMELEN); - if (ops[i].am_error == 0 || ops[i].am_error == MAXNAMELEN) - error = -ERANGE; - if (ops[i].am_error < 0) - break; - - switch (ops[i].am_opcode) { - case ATTR_OP_GET: - ops[i].am_error = xfs_attrmulti_attr_get( - d_inode(dentry), attr_name, - ops[i].am_attrvalue, &ops[i].am_length, - ops[i].am_flags); - break; - case ATTR_OP_SET: - ops[i].am_error = mnt_want_write_file(parfilp); - if (ops[i].am_error) - break; - ops[i].am_error = xfs_attrmulti_attr_set( - d_inode(dentry), attr_name, - ops[i].am_attrvalue, ops[i].am_length, - ops[i].am_flags); - mnt_drop_write_file(parfilp); - break; - case ATTR_OP_REMOVE: - ops[i].am_error = mnt_want_write_file(parfilp); - if (ops[i].am_error) - break; - ops[i].am_error = xfs_attrmulti_attr_remove( - d_inode(dentry), attr_name, - ops[i].am_flags); - mnt_drop_write_file(parfilp); - break; - default: - ops[i].am_error = -EINVAL; - } + ops[i].am_error = xfs_ioc_attrmulti_one(parfilp, + d_inode(dentry), ops[i].am_opcode, + ops[i].am_attrname, ops[i].am_attrvalue, + &ops[i].am_length, ops[i].am_flags); } if (copy_to_user(am_hreq.ops, ops, size)) error = -EFAULT; - kfree(attr_name); - out_kfree_ops: kfree(ops); out_dput: dput(dentry); @@ -1162,7 +1263,7 @@ xfs_ioctl_setattr_xflags( /* diflags2 only valid for v3 inodes. */ di_flags2 = xfs_flags2diflags2(ip, fa->fsx_xflags); - if (di_flags2 && ip->i_d.di_version < 3) + if (di_flags2 && !xfs_sb_version_has_v3inode(&mp->m_sb)) return -EINVAL; ip->i_d.di_flags = xfs_flags2diflags(ip, fa->fsx_xflags); @@ -1372,8 +1473,7 @@ xfs_ioctl_setattr_check_cowextsize( if (!(fa->fsx_xflags & FS_XFLAG_COWEXTSIZE)) return 0; - if (!xfs_sb_version_hasreflink(&ip->i_mount->m_sb) || - ip->i_d.di_version != 3) + if (!xfs_sb_version_hasreflink(&ip->i_mount->m_sb)) return -EINVAL; if (fa->fsx_cowextsize == 0) @@ -1434,9 +1534,9 @@ xfs_ioctl_setattr( * because the i_*dquot fields will get updated anyway. */ if (XFS_IS_QUOTA_ON(mp)) { - code = xfs_qm_vop_dqalloc(ip, ip->i_d.di_uid, - ip->i_d.di_gid, fa->fsx_projid, - XFS_QMOPT_PQUOTA, &udqp, NULL, &pdqp); + code = xfs_qm_vop_dqalloc(ip, VFS_I(ip)->i_uid, + VFS_I(ip)->i_gid, fa->fsx_projid, + XFS_QMOPT_PQUOTA, &udqp, NULL, &pdqp); if (code) return code; } @@ -1501,7 +1601,6 @@ xfs_ioctl_setattr( olddquot = xfs_qm_vop_chown(tp, ip, &ip->i_pdquot, pdqp); } - ASSERT(ip->i_d.di_version > 1); ip->i_d.di_projid = fa->fsx_projid; } @@ -1514,7 +1613,7 @@ xfs_ioctl_setattr( ip->i_d.di_extsize = fa->fsx_extsize >> mp->m_sb.sb_blocklog; else ip->i_d.di_extsize = 0; - if (ip->i_d.di_version == 3 && + if (xfs_sb_version_has_v3inode(&mp->m_sb) && (ip->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE)) ip->i_d.di_cowextsize = fa->fsx_cowextsize >> mp->m_sb.sb_blocklog; diff --git a/fs/xfs/xfs_ioctl.h b/fs/xfs/xfs_ioctl.h index 420bd95dc326..bab6a5a92407 100644 --- a/fs/xfs/xfs_ioctl.h +++ b/fs/xfs/xfs_ioctl.h @@ -6,6 +6,11 @@ #ifndef __XFS_IOCTL_H__ #define __XFS_IOCTL_H__ +struct xfs_bstat; +struct xfs_ibulk; +struct xfs_inogrp; + + extern int xfs_ioc_space( struct file *filp, @@ -30,27 +35,11 @@ xfs_readlink_by_handle( struct file *parfilp, xfs_fsop_handlereq_t *hreq); -extern int -xfs_attrmulti_attr_get( - struct inode *inode, - unsigned char *name, - unsigned char __user *ubuf, - uint32_t *len, - uint32_t flags); - -extern int -xfs_attrmulti_attr_set( - struct inode *inode, - unsigned char *name, - const unsigned char __user *ubuf, - uint32_t len, - uint32_t flags); - -extern int -xfs_attrmulti_attr_remove( - struct inode *inode, - unsigned char *name, - uint32_t flags); +int xfs_ioc_attrmulti_one(struct file *parfilp, struct inode *inode, + uint32_t opcode, void __user *uname, void __user *value, + uint32_t *len, uint32_t flags); +int xfs_ioc_attr_list(struct xfs_inode *dp, void __user *ubuf, int bufsize, + int flags, struct xfs_attrlist_cursor __user *ucursor); extern struct dentry * xfs_handle_to_dentry( @@ -70,10 +59,6 @@ xfs_file_compat_ioctl( unsigned int cmd, unsigned long arg); -struct xfs_ibulk; -struct xfs_bstat; -struct xfs_inogrp; - int xfs_fsbulkstat_one_fmt(struct xfs_ibulk *breq, const struct xfs_bulkstat *bstat); int xfs_fsinumbers_fmt(struct xfs_ibulk *breq, const struct xfs_inumbers *igrp); diff --git a/fs/xfs/xfs_ioctl32.c b/fs/xfs/xfs_ioctl32.c index 769581a79c58..c1771e728117 100644 --- a/fs/xfs/xfs_ioctl32.c +++ b/fs/xfs/xfs_ioctl32.c @@ -352,56 +352,24 @@ xfs_compat_handlereq_to_dentry( STATIC int xfs_compat_attrlist_by_handle( struct file *parfilp, - void __user *arg) + compat_xfs_fsop_attrlist_handlereq_t __user *p) { - int error; - attrlist_cursor_kern_t *cursor; - compat_xfs_fsop_attrlist_handlereq_t __user *p = arg; compat_xfs_fsop_attrlist_handlereq_t al_hreq; struct dentry *dentry; - char *kbuf; + int error; if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (copy_from_user(&al_hreq, arg, - sizeof(compat_xfs_fsop_attrlist_handlereq_t))) + if (copy_from_user(&al_hreq, p, sizeof(al_hreq))) return -EFAULT; - if (al_hreq.buflen < sizeof(struct attrlist) || - al_hreq.buflen > XFS_XATTR_LIST_MAX) - return -EINVAL; - - /* - * Reject flags, only allow namespaces. - */ - if (al_hreq.flags & ~(ATTR_ROOT | ATTR_SECURE)) - return -EINVAL; dentry = xfs_compat_handlereq_to_dentry(parfilp, &al_hreq.hreq); if (IS_ERR(dentry)) return PTR_ERR(dentry); - error = -ENOMEM; - kbuf = kmem_zalloc_large(al_hreq.buflen, 0); - if (!kbuf) - goto out_dput; - - cursor = (attrlist_cursor_kern_t *)&al_hreq.pos; - error = xfs_attr_list(XFS_I(d_inode(dentry)), kbuf, al_hreq.buflen, - al_hreq.flags, cursor); - if (error) - goto out_kfree; - - if (copy_to_user(&p->pos, cursor, sizeof(attrlist_cursor_kern_t))) { - error = -EFAULT; - goto out_kfree; - } - - if (copy_to_user(compat_ptr(al_hreq.buffer), kbuf, al_hreq.buflen)) - error = -EFAULT; - -out_kfree: - kmem_free(kbuf); -out_dput: + error = xfs_ioc_attr_list(XFS_I(d_inode(dentry)), + compat_ptr(al_hreq.buffer), al_hreq.buflen, + al_hreq.flags, &p->pos); dput(dentry); return error; } @@ -416,7 +384,6 @@ xfs_compat_attrmulti_by_handle( compat_xfs_fsop_attrmulti_handlereq_t am_hreq; struct dentry *dentry; unsigned int i, size; - unsigned char *attr_name; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -443,64 +410,18 @@ xfs_compat_attrmulti_by_handle( goto out_dput; } - error = -ENOMEM; - attr_name = kmalloc(MAXNAMELEN, GFP_KERNEL); - if (!attr_name) - goto out_kfree_ops; - error = 0; for (i = 0; i < am_hreq.opcount; i++) { - if ((ops[i].am_flags & ATTR_ROOT) && - (ops[i].am_flags & ATTR_SECURE)) { - ops[i].am_error = -EINVAL; - continue; - } - ops[i].am_flags &= ~ATTR_KERNEL_FLAGS; - - ops[i].am_error = strncpy_from_user((char *)attr_name, + ops[i].am_error = xfs_ioc_attrmulti_one(parfilp, + d_inode(dentry), ops[i].am_opcode, compat_ptr(ops[i].am_attrname), - MAXNAMELEN); - if (ops[i].am_error == 0 || ops[i].am_error == MAXNAMELEN) - error = -ERANGE; - if (ops[i].am_error < 0) - break; - - switch (ops[i].am_opcode) { - case ATTR_OP_GET: - ops[i].am_error = xfs_attrmulti_attr_get( - d_inode(dentry), attr_name, - compat_ptr(ops[i].am_attrvalue), - &ops[i].am_length, ops[i].am_flags); - break; - case ATTR_OP_SET: - ops[i].am_error = mnt_want_write_file(parfilp); - if (ops[i].am_error) - break; - ops[i].am_error = xfs_attrmulti_attr_set( - d_inode(dentry), attr_name, - compat_ptr(ops[i].am_attrvalue), - ops[i].am_length, ops[i].am_flags); - mnt_drop_write_file(parfilp); - break; - case ATTR_OP_REMOVE: - ops[i].am_error = mnt_want_write_file(parfilp); - if (ops[i].am_error) - break; - ops[i].am_error = xfs_attrmulti_attr_remove( - d_inode(dentry), attr_name, - ops[i].am_flags); - mnt_drop_write_file(parfilp); - break; - default: - ops[i].am_error = -EINVAL; - } + compat_ptr(ops[i].am_attrvalue), + &ops[i].am_length, ops[i].am_flags); } if (copy_to_user(compat_ptr(am_hreq.ops), ops, size)) error = -EFAULT; - kfree(attr_name); - out_kfree_ops: kfree(ops); out_dput: dput(dentry); diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 81f2f93caec0..f7a99b3bbcf7 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -22,7 +22,6 @@ #include "xfs_iomap.h" #include "xfs_error.h" -#include #include #include #include @@ -50,10 +49,15 @@ xfs_initxattrs( int error = 0; for (xattr = xattr_array; xattr->name != NULL; xattr++) { - error = xfs_attr_set(ip, xattr->name, - strlen(xattr->name), - xattr->value, xattr->value_len, - ATTR_SECURE); + struct xfs_da_args args = { + .dp = ip, + .attr_filter = XFS_ATTR_SECURE, + .name = xattr->name, + .namelen = strlen(xattr->name), + .value = xattr->value, + .valuelen = xattr->value_len, + }; + error = xfs_attr_set(&args); if (error < 0) break; } @@ -553,7 +557,7 @@ xfs_vn_getattr( stat->blocks = XFS_FSB_TO_BB(mp, ip->i_d.di_nblocks + ip->i_delayed_blks); - if (ip->i_d.di_version == 3) { + if (xfs_sb_version_has_v3inode(&mp->m_sb)) { if (request_mask & STATX_BTIME) { stat->result_mask |= STATX_BTIME; stat->btime = ip->i_d.di_crtime; @@ -692,9 +696,7 @@ xfs_setattr_nonsize( */ ASSERT(udqp == NULL); ASSERT(gdqp == NULL); - error = xfs_qm_vop_dqalloc(ip, xfs_kuid_to_uid(uid), - xfs_kgid_to_gid(gid), - ip->i_d.di_projid, + error = xfs_qm_vop_dqalloc(ip, uid, gid, ip->i_d.di_projid, qflags, &udqp, &gdqp, NULL); if (error) return error; @@ -763,7 +765,6 @@ xfs_setattr_nonsize( olddquot1 = xfs_qm_vop_chown(tp, ip, &ip->i_udquot, udqp); } - ip->i_d.di_uid = xfs_kuid_to_uid(uid); inode->i_uid = uid; } if (!gid_eq(igid, gid)) { @@ -775,7 +776,6 @@ xfs_setattr_nonsize( olddquot2 = xfs_qm_vop_chown(tp, ip, &ip->i_gdquot, gdqp); } - ip->i_d.di_gid = xfs_kgid_to_gid(gid); inode->i_gid = gid; } } @@ -1304,9 +1304,6 @@ xfs_setup_inode( /* make the inode look hashed for the writeback code */ inode_fake_hash(inode); - inode->i_uid = xfs_uid_to_kuid(ip->i_d.di_uid); - inode->i_gid = xfs_gid_to_kgid(ip->i_d.di_gid); - i_size_write(inode, ip->i_d.di_size); xfs_diflags_to_iflags(inode, ip); diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index 4b31c29b7e6b..ff2da28fed90 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -86,8 +86,8 @@ xfs_bulkstat_one_int( */ buf->bs_projectid = ip->i_d.di_projid; buf->bs_ino = ino; - buf->bs_uid = dic->di_uid; - buf->bs_gid = dic->di_gid; + buf->bs_uid = i_uid_read(inode); + buf->bs_gid = i_gid_read(inode); buf->bs_size = dic->di_size; buf->bs_nlink = inode->i_nlink; @@ -110,7 +110,7 @@ xfs_bulkstat_one_int( buf->bs_forkoff = XFS_IFORK_BOFF(ip); buf->bs_version = XFS_BULKSTAT_VERSION_V5; - if (dic->di_version == 3) { + if (xfs_sb_version_has_v3inode(&mp->m_sb)) { if (dic->di_flags2 & XFS_DIFLAG2_COWEXTSIZE) buf->bs_cowextsize_blks = dic->di_cowextsize; } diff --git a/fs/xfs/xfs_linux.h b/fs/xfs/xfs_linux.h index 8738bb03f253..9f70d2f68e05 100644 --- a/fs/xfs/xfs_linux.h +++ b/fs/xfs/xfs_linux.h @@ -60,6 +60,7 @@ typedef __u32 xfs_nlink_t; #include #include #include +#include #include #include @@ -163,32 +164,6 @@ struct xstats { extern struct xstats xfsstats; -/* Kernel uid/gid conversion. These are used to convert to/from the on disk - * uid_t/gid_t types to the kuid_t/kgid_t types that the kernel uses internally. - * The conversion here is type only, the value will remain the same since we - * are converting to the init_user_ns. The uid is later mapped to a particular - * user namespace value when crossing the kernel/user boundary. - */ -static inline uint32_t xfs_kuid_to_uid(kuid_t uid) -{ - return from_kuid(&init_user_ns, uid); -} - -static inline kuid_t xfs_uid_to_kuid(uint32_t uid) -{ - return make_kuid(&init_user_ns, uid); -} - -static inline uint32_t xfs_kgid_to_gid(kgid_t gid) -{ - return from_kgid(&init_user_ns, gid); -} - -static inline kgid_t xfs_gid_to_kgid(uint32_t gid) -{ - return make_kgid(&init_user_ns, gid); -} - static inline dev_t xfs_to_linux_dev_t(xfs_dev_t dev) { return MKDEV(sysv_major(dev) & 0x1ff, sysv_minor(dev)); diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index f6006d94a581..4a53768c5397 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -47,8 +47,7 @@ xlog_dealloc_log( /* local state machine functions */ STATIC void xlog_state_done_syncing( - struct xlog_in_core *iclog, - bool aborted); + struct xlog_in_core *iclog); STATIC int xlog_state_get_iclog_space( struct xlog *log, @@ -63,11 +62,6 @@ xlog_state_switch_iclogs( struct xlog_in_core *iclog, int eventual_size); STATIC void -xlog_state_want_sync( - struct xlog *log, - struct xlog_in_core *iclog); - -STATIC void xlog_grant_push_ail( struct xlog *log, int need_bytes); @@ -597,26 +591,21 @@ xlog_state_release_iclog( return 0; } -int +void xfs_log_release_iclog( - struct xfs_mount *mp, struct xlog_in_core *iclog) { - struct xlog *log = mp->m_log; - bool sync; - - if (iclog->ic_state == XLOG_STATE_IOERROR) { - xfs_force_shutdown(mp, SHUTDOWN_LOG_IO_ERROR); - return -EIO; - } + struct xlog *log = iclog->ic_log; + bool sync = false; if (atomic_dec_and_lock(&iclog->ic_refcnt, &log->l_icloglock)) { - sync = __xlog_state_release_iclog(log, iclog); + if (iclog->ic_state != XLOG_STATE_IOERROR) + sync = __xlog_state_release_iclog(log, iclog); spin_unlock(&log->l_icloglock); - if (sync) - xlog_sync(log, iclog); } - return 0; + + if (sync) + xlog_sync(log, iclog); } /* @@ -854,6 +843,31 @@ xfs_log_mount_cancel( xfs_log_unmount(mp); } +/* + * Wait for the iclog to be written disk, or return an error if the log has been + * shut down. + */ +static int +xlog_wait_on_iclog( + struct xlog_in_core *iclog) + __releases(iclog->ic_log->l_icloglock) +{ + struct xlog *log = iclog->ic_log; + + if (!XLOG_FORCED_SHUTDOWN(log) && + iclog->ic_state != XLOG_STATE_ACTIVE && + iclog->ic_state != XLOG_STATE_DIRTY) { + XFS_STATS_INC(log->l_mp, xs_log_force_sleep); + xlog_wait(&iclog->ic_force_wait, &log->l_icloglock); + } else { + spin_unlock(&log->l_icloglock); + } + + if (XLOG_FORCED_SHUTDOWN(log)) + return -EIO; + return 0; +} + /* * Final log writes as part of unmount. * @@ -919,20 +933,13 @@ xfs_log_write_unmount_record( spin_lock(&log->l_icloglock); iclog = log->l_iclog; atomic_inc(&iclog->ic_refcnt); - xlog_state_want_sync(log, iclog); + if (iclog->ic_state == XLOG_STATE_ACTIVE) + xlog_state_switch_iclogs(log, iclog, 0); + else + ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC || + iclog->ic_state == XLOG_STATE_IOERROR); error = xlog_state_release_iclog(log, iclog); - switch (iclog->ic_state) { - default: - if (!XLOG_FORCED_SHUTDOWN(log)) { - xlog_wait(&iclog->ic_force_wait, &log->l_icloglock); - break; - } - /* fall through */ - case XLOG_STATE_ACTIVE: - case XLOG_STATE_DIRTY: - spin_unlock(&log->l_icloglock); - break; - } + xlog_wait_on_iclog(iclog); if (tic) { trace_xfs_log_umount_write(log, tic); @@ -941,6 +948,18 @@ xfs_log_write_unmount_record( } } +static void +xfs_log_unmount_verify_iclog( + struct xlog *log) +{ + struct xlog_in_core *iclog = log->l_iclog; + + do { + ASSERT(iclog->ic_state == XLOG_STATE_ACTIVE); + ASSERT(iclog->ic_offset == 0); + } while ((iclog = iclog->ic_next) != log->l_iclog); +} + /* * Unmount record used to have a string "Unmount filesystem--" in the * data section where the "Un" was really a magic number (XLOG_UNMOUNT_TYPE). @@ -948,16 +967,11 @@ xfs_log_write_unmount_record( * currently architecture converted and "Unmount" is a bit foo. * As far as I know, there weren't any dependencies on the old behaviour. */ - -static int -xfs_log_unmount_write(xfs_mount_t *mp) +static void +xfs_log_unmount_write( + struct xfs_mount *mp) { - struct xlog *log = mp->m_log; - xlog_in_core_t *iclog; -#ifdef DEBUG - xlog_in_core_t *first_iclog; -#endif - int error; + struct xlog *log = mp->m_log; /* * Don't write out unmount record on norecovery mounts or ro devices. @@ -966,57 +980,16 @@ xfs_log_unmount_write(xfs_mount_t *mp) if (mp->m_flags & XFS_MOUNT_NORECOVERY || xfs_readonly_buftarg(log->l_targ)) { ASSERT(mp->m_flags & XFS_MOUNT_RDONLY); - return 0; + return; } - error = xfs_log_force(mp, XFS_LOG_SYNC); - ASSERT(error || !(XLOG_FORCED_SHUTDOWN(log))); + xfs_log_force(mp, XFS_LOG_SYNC); -#ifdef DEBUG - first_iclog = iclog = log->l_iclog; - do { - if (iclog->ic_state != XLOG_STATE_IOERROR) { - ASSERT(iclog->ic_state == XLOG_STATE_ACTIVE); - ASSERT(iclog->ic_offset == 0); - } - iclog = iclog->ic_next; - } while (iclog != first_iclog); -#endif - if (! (XLOG_FORCED_SHUTDOWN(log))) { - xfs_log_write_unmount_record(mp); - } else { - /* - * We're already in forced_shutdown mode, couldn't - * even attempt to write out the unmount transaction. - * - * Go through the motions of sync'ing and releasing - * the iclog, even though no I/O will actually happen, - * we need to wait for other log I/Os that may already - * be in progress. Do this as a separate section of - * code so we'll know if we ever get stuck here that - * we're in this odd situation of trying to unmount - * a file system that went into forced_shutdown as - * the result of an unmount.. - */ - spin_lock(&log->l_icloglock); - iclog = log->l_iclog; - atomic_inc(&iclog->ic_refcnt); - xlog_state_want_sync(log, iclog); - error = xlog_state_release_iclog(log, iclog); - switch (iclog->ic_state) { - case XLOG_STATE_ACTIVE: - case XLOG_STATE_DIRTY: - case XLOG_STATE_IOERROR: - spin_unlock(&log->l_icloglock); - break; - default: - xlog_wait(&iclog->ic_force_wait, &log->l_icloglock); - break; - } - } - - return error; -} /* xfs_log_unmount_write */ + if (XLOG_FORCED_SHUTDOWN(log)) + return; + xfs_log_unmount_verify_iclog(log); + xfs_log_write_unmount_record(mp); +} /* * Empty the log for unmount/freeze. @@ -1279,7 +1252,6 @@ xlog_ioend_work( struct xlog_in_core *iclog = container_of(work, struct xlog_in_core, ic_end_io_work); struct xlog *log = iclog->ic_log; - bool aborted = false; int error; error = blk_status_to_errno(iclog->ic_bio.bi_status); @@ -1295,17 +1267,9 @@ xlog_ioend_work( if (XFS_TEST_ERROR(error, log->l_mp, XFS_ERRTAG_IODONE_IOERR)) { xfs_alert(log->l_mp, "log I/O error %d", error); xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR); - /* - * This flag will be propagated to the trans-committed - * callback routines to let them know that the log-commit - * didn't succeed. - */ - aborted = true; - } else if (iclog->ic_state == XLOG_STATE_IOERROR) { - aborted = true; } - xlog_state_done_syncing(iclog, aborted); + xlog_state_done_syncing(iclog); bio_uninit(&iclog->ic_bio); /* @@ -1739,7 +1703,7 @@ xlog_bio_end_io( &iclog->ic_end_io_work); } -static void +static int xlog_map_iclog_data( struct bio *bio, void *data, @@ -1750,11 +1714,14 @@ xlog_map_iclog_data( unsigned int off = offset_in_page(data); size_t len = min_t(size_t, count, PAGE_SIZE - off); - WARN_ON_ONCE(bio_add_page(bio, page, len, off) != len); + if (bio_add_page(bio, page, len, off) != len) + return -EIO; data += len; count -= len; } while (count); + + return 0; } STATIC void @@ -1784,7 +1751,7 @@ xlog_write_iclog( * the buffer manually, the code needs to be kept in sync * with the I/O completion path. */ - xlog_state_done_syncing(iclog, true); + xlog_state_done_syncing(iclog); up(&iclog->ic_sema); return; } @@ -1798,7 +1765,10 @@ xlog_write_iclog( if (need_flush) iclog->ic_bio.bi_opf |= REQ_PREFLUSH; - xlog_map_iclog_data(&iclog->ic_bio, iclog->ic_data, count); + if (xlog_map_iclog_data(&iclog->ic_bio, iclog->ic_data, count)) { + xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR); + return; + } if (is_vmalloc_addr(iclog->ic_data)) flush_kernel_vmap_range(iclog->ic_data, count); @@ -2328,7 +2298,11 @@ xlog_write_copy_finish( *record_cnt = 0; *data_cnt = 0; - xlog_state_want_sync(log, iclog); + if (iclog->ic_state == XLOG_STATE_ACTIVE) + xlog_state_switch_iclogs(log, iclog, 0); + else + ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC || + iclog->ic_state == XLOG_STATE_IOERROR); if (!commit_iclog) goto release_iclog; spin_unlock(&log->l_icloglock); @@ -2575,111 +2549,106 @@ xlog_write( ***************************************************************************** */ +static void +xlog_state_activate_iclog( + struct xlog_in_core *iclog, + int *iclogs_changed) +{ + ASSERT(list_empty_careful(&iclog->ic_callbacks)); + + /* + * If the number of ops in this iclog indicate it just contains the + * dummy transaction, we can change state into IDLE (the second time + * around). Otherwise we should change the state into NEED a dummy. + * We don't need to cover the dummy. + */ + if (*iclogs_changed == 0 && + iclog->ic_header.h_num_logops == cpu_to_be32(XLOG_COVER_OPS)) { + *iclogs_changed = 1; + } else { + /* + * We have two dirty iclogs so start over. This could also be + * num of ops indicating this is not the dummy going out. + */ + *iclogs_changed = 2; + } + + iclog->ic_state = XLOG_STATE_ACTIVE; + iclog->ic_offset = 0; + iclog->ic_header.h_num_logops = 0; + memset(iclog->ic_header.h_cycle_data, 0, + sizeof(iclog->ic_header.h_cycle_data)); + iclog->ic_header.h_lsn = 0; +} + /* - * An iclog has just finished IO completion processing, so we need to update - * the iclog state and propagate that up into the overall log state. Hence we - * prepare the iclog for cleaning, and then clean all the pending dirty iclogs - * starting from the head, and then wake up any threads that are waiting for the - * iclog to be marked clean. - * - * The ordering of marking iclogs ACTIVE must be maintained, so an iclog - * doesn't become ACTIVE beyond one that is SYNCING. This is also required to - * maintain the notion that we use a ordered wait queue to hold off would be - * writers to the log when every iclog is trying to sync to disk. - * - * Caller must hold the icloglock before calling us. - * - * State Change: !IOERROR -> DIRTY -> ACTIVE + * Loop through all iclogs and mark all iclogs currently marked DIRTY as + * ACTIVE after iclog I/O has completed. */ +static void +xlog_state_activate_iclogs( + struct xlog *log, + int *iclogs_changed) +{ + struct xlog_in_core *iclog = log->l_iclog; + + do { + if (iclog->ic_state == XLOG_STATE_DIRTY) + xlog_state_activate_iclog(iclog, iclogs_changed); + /* + * The ordering of marking iclogs ACTIVE must be maintained, so + * an iclog doesn't become ACTIVE beyond one that is SYNCING. + */ + else if (iclog->ic_state != XLOG_STATE_ACTIVE) + break; + } while ((iclog = iclog->ic_next) != log->l_iclog); +} + +static int +xlog_covered_state( + int prev_state, + int iclogs_changed) +{ + /* + * We usually go to NEED. But we go to NEED2 if the changed indicates we + * are done writing the dummy record. If we are done with the second + * dummy recored (DONE2), then we go to IDLE. + */ + switch (prev_state) { + case XLOG_STATE_COVER_IDLE: + case XLOG_STATE_COVER_NEED: + case XLOG_STATE_COVER_NEED2: + break; + case XLOG_STATE_COVER_DONE: + if (iclogs_changed == 1) + return XLOG_STATE_COVER_NEED2; + break; + case XLOG_STATE_COVER_DONE2: + if (iclogs_changed == 1) + return XLOG_STATE_COVER_IDLE; + break; + default: + ASSERT(0); + } + + return XLOG_STATE_COVER_NEED; +} + STATIC void xlog_state_clean_iclog( struct xlog *log, struct xlog_in_core *dirty_iclog) { - struct xlog_in_core *iclog; - int changed = 0; + int iclogs_changed = 0; - /* Prepare the completed iclog. */ - if (dirty_iclog->ic_state != XLOG_STATE_IOERROR) - dirty_iclog->ic_state = XLOG_STATE_DIRTY; + dirty_iclog->ic_state = XLOG_STATE_DIRTY; - /* Walk all the iclogs to update the ordered active state. */ - iclog = log->l_iclog; - do { - if (iclog->ic_state == XLOG_STATE_DIRTY) { - iclog->ic_state = XLOG_STATE_ACTIVE; - iclog->ic_offset = 0; - ASSERT(list_empty_careful(&iclog->ic_callbacks)); - /* - * If the number of ops in this iclog indicate it just - * contains the dummy transaction, we can - * change state into IDLE (the second time around). - * Otherwise we should change the state into - * NEED a dummy. - * We don't need to cover the dummy. - */ - if (!changed && - (be32_to_cpu(iclog->ic_header.h_num_logops) == - XLOG_COVER_OPS)) { - changed = 1; - } else { - /* - * We have two dirty iclogs so start over - * This could also be num of ops indicates - * this is not the dummy going out. - */ - changed = 2; - } - iclog->ic_header.h_num_logops = 0; - memset(iclog->ic_header.h_cycle_data, 0, - sizeof(iclog->ic_header.h_cycle_data)); - iclog->ic_header.h_lsn = 0; - } else if (iclog->ic_state == XLOG_STATE_ACTIVE) - /* do nothing */; - else - break; /* stop cleaning */ - iclog = iclog->ic_next; - } while (iclog != log->l_iclog); - - - /* - * Wake up threads waiting in xfs_log_force() for the dirty iclog - * to be cleaned. - */ + xlog_state_activate_iclogs(log, &iclogs_changed); wake_up_all(&dirty_iclog->ic_force_wait); - /* - * Change state for the dummy log recording. - * We usually go to NEED. But we go to NEED2 if the changed indicates - * we are done writing the dummy record. - * If we are done with the second dummy recored (DONE2), then - * we go to IDLE. - */ - if (changed) { - switch (log->l_covered_state) { - case XLOG_STATE_COVER_IDLE: - case XLOG_STATE_COVER_NEED: - case XLOG_STATE_COVER_NEED2: - log->l_covered_state = XLOG_STATE_COVER_NEED; - break; - - case XLOG_STATE_COVER_DONE: - if (changed == 1) - log->l_covered_state = XLOG_STATE_COVER_NEED2; - else - log->l_covered_state = XLOG_STATE_COVER_NEED; - break; - - case XLOG_STATE_COVER_DONE2: - if (changed == 1) - log->l_covered_state = XLOG_STATE_COVER_IDLE; - else - log->l_covered_state = XLOG_STATE_COVER_NEED; - break; - - default: - ASSERT(0); - } + if (iclogs_changed) { + log->l_covered_state = xlog_covered_state(log->l_covered_state, + iclogs_changed); } } @@ -2808,8 +2777,7 @@ xlog_state_iodone_process_iclog( static void xlog_state_do_iclog_callbacks( struct xlog *log, - struct xlog_in_core *iclog, - bool aborted) + struct xlog_in_core *iclog) __releases(&log->l_icloglock) __acquires(&log->l_icloglock) { @@ -2821,7 +2789,7 @@ xlog_state_do_iclog_callbacks( list_splice_init(&iclog->ic_callbacks, &tmp); spin_unlock(&iclog->ic_callback_lock); - xlog_cil_process_committed(&tmp, aborted); + xlog_cil_process_committed(&tmp); spin_lock(&iclog->ic_callback_lock); } @@ -2836,8 +2804,7 @@ xlog_state_do_iclog_callbacks( STATIC void xlog_state_do_callback( - struct xlog *log, - bool aborted) + struct xlog *log) { struct xlog_in_core *iclog; struct xlog_in_core *first_iclog; @@ -2878,9 +2845,11 @@ xlog_state_do_callback( * we'll have to run at least one more complete loop. */ cycled_icloglock = true; - xlog_state_do_iclog_callbacks(log, iclog, aborted); - - xlog_state_clean_iclog(log, iclog); + xlog_state_do_iclog_callbacks(log, iclog); + if (XLOG_FORCED_SHUTDOWN(log)) + wake_up_all(&iclog->ic_force_wait); + else + xlog_state_clean_iclog(log, iclog); iclog = iclog->ic_next; } while (first_iclog != iclog); @@ -2916,25 +2885,22 @@ xlog_state_do_callback( */ STATIC void xlog_state_done_syncing( - struct xlog_in_core *iclog, - bool aborted) + struct xlog_in_core *iclog) { struct xlog *log = iclog->ic_log; spin_lock(&log->l_icloglock); - ASSERT(atomic_read(&iclog->ic_refcnt) == 0); /* * If we got an error, either on the first buffer, or in the case of - * split log writes, on the second, we mark ALL iclogs STATE_IOERROR, - * and none should ever be attempted to be written to disk - * again. + * split log writes, on the second, we shut down the file system and + * no iclogs should ever be attempted to be written to disk again. */ - if (iclog->ic_state == XLOG_STATE_SYNCING) + if (!XLOG_FORCED_SHUTDOWN(log)) { + ASSERT(iclog->ic_state == XLOG_STATE_SYNCING); iclog->ic_state = XLOG_STATE_DONE_SYNC; - else - ASSERT(iclog->ic_state == XLOG_STATE_IOERROR); + } /* * Someone could be sleeping prior to writing out the next @@ -2943,9 +2909,8 @@ xlog_state_done_syncing( */ wake_up_all(&iclog->ic_write_wait); spin_unlock(&log->l_icloglock); - xlog_state_do_callback(log, aborted); /* also cleans log */ -} /* xlog_state_done_syncing */ - + xlog_state_do_callback(log); /* also cleans log */ +} /* * If the head of the in-core log ring is not (ACTIVE or DIRTY), then we must @@ -3152,11 +3117,12 @@ xlog_ungrant_log_space( } /* - * This routine will mark the current iclog in the ring as WANT_SYNC - * and move the current iclog pointer to the next iclog in the ring. - * When this routine is called from xlog_state_get_iclog_space(), the - * exact size of the iclog has not yet been determined. All we know is - * that every data block. We have run out of space in this log record. + * Mark the current iclog in the ring as WANT_SYNC and move the current iclog + * pointer to the next iclog in the ring. + * + * When called from xlog_state_get_iclog_space(), the exact size of the iclog + * has not yet been determined, all we know is that we have run out of space in + * the current iclog. */ STATIC void xlog_state_switch_iclogs( @@ -3165,6 +3131,8 @@ xlog_state_switch_iclogs( int eventual_size) { ASSERT(iclog->ic_state == XLOG_STATE_ACTIVE); + assert_spin_locked(&log->l_icloglock); + if (!eventual_size) eventual_size = iclog->ic_offset; iclog->ic_state = XLOG_STATE_WANT_SYNC; @@ -3259,9 +3227,6 @@ xfs_log_force( * previous iclog and go to sleep. */ iclog = iclog->ic_prev; - if (iclog->ic_state == XLOG_STATE_ACTIVE || - iclog->ic_state == XLOG_STATE_DIRTY) - goto out_unlock; } else if (iclog->ic_state == XLOG_STATE_ACTIVE) { if (atomic_read(&iclog->ic_refcnt) == 0) { /* @@ -3277,8 +3242,7 @@ xfs_log_force( if (xlog_state_release_iclog(log, iclog)) goto out_error; - if (be64_to_cpu(iclog->ic_header.h_lsn) != lsn || - iclog->ic_state == XLOG_STATE_DIRTY) + if (be64_to_cpu(iclog->ic_header.h_lsn) != lsn) goto out_unlock; } else { /* @@ -3298,17 +3262,8 @@ xfs_log_force( ; } - if (!(flags & XFS_LOG_SYNC)) - goto out_unlock; - - if (iclog->ic_state == XLOG_STATE_IOERROR) - goto out_error; - XFS_STATS_INC(mp, xs_log_force_sleep); - xlog_wait(&iclog->ic_force_wait, &log->l_icloglock); - if (iclog->ic_state == XLOG_STATE_IOERROR) - return -EIO; - return 0; - + if (flags & XFS_LOG_SYNC) + return xlog_wait_on_iclog(iclog); out_unlock: spin_unlock(&log->l_icloglock); return 0; @@ -3339,9 +3294,6 @@ __xfs_log_force_lsn( goto out_unlock; } - if (iclog->ic_state == XLOG_STATE_DIRTY) - goto out_unlock; - if (iclog->ic_state == XLOG_STATE_ACTIVE) { /* * We sleep here if we haven't already slept (e.g. this is the @@ -3375,20 +3327,8 @@ __xfs_log_force_lsn( *log_flushed = 1; } - if (!(flags & XFS_LOG_SYNC) || - (iclog->ic_state == XLOG_STATE_ACTIVE || - iclog->ic_state == XLOG_STATE_DIRTY)) - goto out_unlock; - - if (iclog->ic_state == XLOG_STATE_IOERROR) - goto out_error; - - XFS_STATS_INC(mp, xs_log_force_sleep); - xlog_wait(&iclog->ic_force_wait, &log->l_icloglock); - if (iclog->ic_state == XLOG_STATE_IOERROR) - return -EIO; - return 0; - + if (flags & XFS_LOG_SYNC) + return xlog_wait_on_iclog(iclog); out_unlock: spin_unlock(&log->l_icloglock); return 0; @@ -3434,26 +3374,6 @@ xfs_log_force_lsn( return ret; } -/* - * Called when we want to mark the current iclog as being ready to sync to - * disk. - */ -STATIC void -xlog_state_want_sync( - struct xlog *log, - struct xlog_in_core *iclog) -{ - assert_spin_locked(&log->l_icloglock); - - if (iclog->ic_state == XLOG_STATE_ACTIVE) { - xlog_state_switch_iclogs(log, iclog, 0); - } else { - ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC || - iclog->ic_state == XLOG_STATE_IOERROR); - } -} - - /***************************************************************************** * * TICKET functions @@ -3937,7 +3857,7 @@ xfs_log_force_umount( spin_lock(&log->l_cilp->xc_push_lock); wake_up_all(&log->l_cilp->xc_commit_wait); spin_unlock(&log->l_cilp->xc_push_lock); - xlog_state_do_callback(log, true); + xlog_state_do_callback(log); /* return non-zero if log IOERROR transition had already happened */ return retval; diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h index 84e06805160f..cc77cc36560a 100644 --- a/fs/xfs/xfs_log.h +++ b/fs/xfs/xfs_log.h @@ -121,8 +121,7 @@ void xfs_log_mount_cancel(struct xfs_mount *); xfs_lsn_t xlog_assign_tail_lsn(struct xfs_mount *mp); xfs_lsn_t xlog_assign_tail_lsn_locked(struct xfs_mount *mp); void xfs_log_space_wake(struct xfs_mount *mp); -int xfs_log_release_iclog(struct xfs_mount *mp, - struct xlog_in_core *iclog); +void xfs_log_release_iclog(struct xlog_in_core *iclog); int xfs_log_reserve(struct xfs_mount *mp, int length, int count, @@ -138,7 +137,7 @@ void xfs_log_ticket_put(struct xlog_ticket *ticket); void xfs_log_commit_cil(struct xfs_mount *mp, struct xfs_trans *tp, xfs_lsn_t *commit_lsn, bool regrant); -void xlog_cil_process_committed(struct list_head *list, bool aborted); +void xlog_cil_process_committed(struct list_head *list); bool xfs_log_item_in_current_chkpt(struct xfs_log_item *lip); void xfs_log_work_queue(struct xfs_mount *mp); diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c index 48435cf2aa16..64cc0bf2ab3b 100644 --- a/fs/xfs/xfs_log_cil.c +++ b/fs/xfs/xfs_log_cil.c @@ -574,10 +574,10 @@ xlog_discard_busy_extents( */ static void xlog_cil_committed( - struct xfs_cil_ctx *ctx, - bool abort) + struct xfs_cil_ctx *ctx) { struct xfs_mount *mp = ctx->cil->xc_log->l_mp; + bool abort = XLOG_FORCED_SHUTDOWN(ctx->cil->xc_log); /* * If the I/O failed, we're aborting the commit and already shutdown. @@ -613,37 +613,38 @@ xlog_cil_committed( void xlog_cil_process_committed( - struct list_head *list, - bool aborted) + struct list_head *list) { struct xfs_cil_ctx *ctx; while ((ctx = list_first_entry_or_null(list, struct xfs_cil_ctx, iclog_entry))) { list_del(&ctx->iclog_entry); - xlog_cil_committed(ctx, aborted); + xlog_cil_committed(ctx); } } /* - * Push the Committed Item List to the log. If @push_seq flag is zero, then it - * is a background flush and so we can chose to ignore it. Otherwise, if the - * current sequence is the same as @push_seq we need to do a flush. If - * @push_seq is less than the current sequence, then it has already been + * Push the Committed Item List to the log. + * + * If the current sequence is the same as xc_push_seq we need to do a flush. If + * xc_push_seq is less than the current sequence, then it has already been * flushed and we don't need to do anything - the caller will wait for it to * complete if necessary. * - * @push_seq is a value rather than a flag because that allows us to do an - * unlocked check of the sequence number for a match. Hence we can allows log - * forces to run racily and not issue pushes for the same sequence twice. If we - * get a race between multiple pushes for the same sequence they will block on - * the first one and then abort, hence avoiding needless pushes. + * xc_push_seq is checked unlocked against the sequence number for a match. + * Hence we can allow log forces to run racily and not issue pushes for the + * same sequence twice. If we get a race between multiple pushes for the same + * sequence they will block on the first one and then abort, hence avoiding + * needless pushes. */ -STATIC int -xlog_cil_push( - struct xlog *log) +static void +xlog_cil_push_work( + struct work_struct *work) { - struct xfs_cil *cil = log->l_cilp; + struct xfs_cil *cil = + container_of(work, struct xfs_cil, xc_push_work); + struct xlog *log = cil->xc_log; struct xfs_log_vec *lv; struct xfs_cil_ctx *ctx; struct xfs_cil_ctx *new_ctx; @@ -657,9 +658,6 @@ xlog_cil_push( xfs_lsn_t commit_lsn; xfs_lsn_t push_seq; - if (!cil) - return 0; - new_ctx = kmem_zalloc(sizeof(*new_ctx), KM_NOFS); new_ctx->ticket = xlog_cil_ticket_alloc(log); @@ -867,28 +865,20 @@ xlog_cil_push( spin_unlock(&cil->xc_push_lock); /* release the hounds! */ - return xfs_log_release_iclog(log->l_mp, commit_iclog); + xfs_log_release_iclog(commit_iclog); + return; out_skip: up_write(&cil->xc_ctx_lock); xfs_log_ticket_put(new_ctx->ticket); kmem_free(new_ctx); - return 0; + return; out_abort_free_ticket: xfs_log_ticket_put(tic); out_abort: - xlog_cil_committed(ctx, true); - return -EIO; -} - -static void -xlog_cil_push_work( - struct work_struct *work) -{ - struct xfs_cil *cil = container_of(work, struct xfs_cil, - xc_push_work); - xlog_cil_push(cil->xc_log); + ASSERT(XLOG_FORCED_SHUTDOWN(log)); + xlog_cil_committed(ctx); } /* diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h index b192c5a9f9fd..2b0aec37e73e 100644 --- a/fs/xfs/xfs_log_priv.h +++ b/fs/xfs/xfs_log_priv.h @@ -402,7 +402,8 @@ struct xlog { #define XLOG_BUF_CANCEL_BUCKET(log, blkno) \ ((log)->l_buf_cancel_table + ((uint64_t)blkno % XLOG_BC_TABLE_SIZE)) -#define XLOG_FORCED_SHUTDOWN(log) ((log)->l_flags & XLOG_IO_ERROR) +#define XLOG_FORCED_SHUTDOWN(log) \ + (unlikely((log)->l_flags & XLOG_IO_ERROR)) /* common routines */ extern int @@ -524,12 +525,6 @@ xlog_cil_force(struct xlog *log) xlog_cil_force_lsn(log, log->l_cilp->xc_current_sequence); } -/* - * Unmount record type is used as a pseudo transaction type for the ticket. - * It's value must be outside the range of XFS_TRANS_* values. - */ -#define XLOG_UNMOUNT_REC_TYPE (-1U) - /* * Wrapper function for waiting on a wait queue serialised against wakeups * by a spinlock. This matches the semantics of all the wait queues used in the diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 25cfc85dbaa7..11c3502b07b1 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2869,8 +2869,8 @@ xfs_recover_inode_owner_change( return -ENOMEM; /* instantiate the inode */ + ASSERT(dip->di_version >= 3); xfs_inode_from_disk(ip, dip); - ASSERT(ip->i_d.di_version >= 3); error = xfs_iformat_fork(ip, dip); if (error) @@ -2997,7 +2997,7 @@ xlog_recover_inode_pass2( * superblock flag to determine whether we need to look at di_flushiter * to skip replay when the on disk inode is newer than the log one */ - if (!xfs_sb_version_hascrc(&mp->m_sb) && + if (!xfs_sb_version_has_v3inode(&mp->m_sb) && ldip->di_flushiter < be16_to_cpu(dip->di_flushiter)) { /* * Deal with the wrap case, DI_MAX_FLUSH is less @@ -3068,7 +3068,7 @@ xlog_recover_inode_pass2( error = -EFSCORRUPTED; goto out_release; } - isize = xfs_log_dinode_size(ldip->di_version); + isize = xfs_log_dinode_size(mp); if (unlikely(item->ri_buf[1].i_len > isize)) { XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(7)", XFS_ERRLEVEL_LOW, mp, ldip, @@ -4947,7 +4947,7 @@ xlog_recover_clear_agi_bucket( if (error) goto out_abort; - agi = XFS_BUF_TO_AGI(agibp); + agi = agibp->b_addr; agi->agi_unlinked[bucket] = cpu_to_be32(NULLAGINO); offset = offsetof(xfs_agi_t, agi_unlinked) + (sizeof(xfs_agino_t) * bucket); @@ -5083,7 +5083,7 @@ xlog_recover_process_iunlinks( * buffer reference though, so that it stays pinned in memory * while we need the buffer. */ - agi = XFS_BUF_TO_AGI(agibp); + agi = agibp->b_addr; xfs_buf_unlock(agibp); for (bucket = 0; bucket < XFS_AGI_UNLINKED_BUCKETS; bucket++) { @@ -5636,7 +5636,7 @@ xlog_do_recover( /* Convert superblock from on-disk format */ sbp = &mp->m_sb; - xfs_sb_from_disk(sbp, XFS_BUF_TO_SBP(bp)); + xfs_sb_from_disk(sbp, bp->b_addr); xfs_buf_relse(bp); /* re-initialise in-core superblock and geometry structures */ @@ -5809,7 +5809,6 @@ xlog_recover_check_summary( struct xlog *log) { xfs_mount_t *mp; - xfs_agf_t *agfp; xfs_buf_t *agfbp; xfs_buf_t *agibp; xfs_agnumber_t agno; @@ -5829,7 +5828,8 @@ xlog_recover_check_summary( xfs_alert(mp, "%s agf read failed agno %d error %d", __func__, agno, error); } else { - agfp = XFS_BUF_TO_AGF(agfbp); + struct xfs_agf *agfp = agfbp->b_addr; + freeblks += be32_to_cpu(agfp->agf_freeblks) + be32_to_cpu(agfp->agf_flcount); xfs_buf_relse(agfbp); @@ -5840,7 +5840,7 @@ xlog_recover_check_summary( xfs_alert(mp, "%s agi read failed agno %d error %d", __func__, agno, error); } else { - struct xfs_agi *agi = XFS_BUF_TO_AGI(agibp); + struct xfs_agi *agi = agibp->b_addr; itotal += be32_to_cpu(agi->agi_count); ifree += be32_to_cpu(agi->agi_freecount); diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 56efe140c923..c5513e5a226a 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -310,7 +310,7 @@ xfs_readsb( /* * Initialize the mount structure from the superblock. */ - xfs_sb_from_disk(sbp, XFS_BUF_TO_SBP(bp)); + xfs_sb_from_disk(sbp, bp->b_addr); /* * If we haven't validated the superblock, do so now before we try diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 0b0909657bad..cabdb755adae 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -326,16 +326,16 @@ xfs_qm_dqattach_locked( ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); if (XFS_IS_UQUOTA_ON(mp) && !ip->i_udquot) { - error = xfs_qm_dqattach_one(ip, ip->i_d.di_uid, XFS_DQ_USER, - doalloc, &ip->i_udquot); + error = xfs_qm_dqattach_one(ip, i_uid_read(VFS_I(ip)), + XFS_DQ_USER, doalloc, &ip->i_udquot); if (error) goto done; ASSERT(ip->i_udquot); } if (XFS_IS_GQUOTA_ON(mp) && !ip->i_gdquot) { - error = xfs_qm_dqattach_one(ip, ip->i_d.di_gid, XFS_DQ_GROUP, - doalloc, &ip->i_gdquot); + error = xfs_qm_dqattach_one(ip, i_gid_read(VFS_I(ip)), + XFS_DQ_GROUP, doalloc, &ip->i_gdquot); if (error) goto done; ASSERT(ip->i_gdquot); @@ -871,12 +871,20 @@ xfs_qm_reset_dqcounts( ddq->d_bcount = 0; ddq->d_icount = 0; ddq->d_rtbcount = 0; - ddq->d_btimer = 0; - ddq->d_itimer = 0; - ddq->d_rtbtimer = 0; - ddq->d_bwarns = 0; - ddq->d_iwarns = 0; - ddq->d_rtbwarns = 0; + + /* + * dquot id 0 stores the default grace period and the maximum + * warning limit that were set by the administrator, so we + * should not reset them. + */ + if (ddq->d_id != 0) { + ddq->d_btimer = 0; + ddq->d_itimer = 0; + ddq->d_rtbtimer = 0; + ddq->d_bwarns = 0; + ddq->d_iwarns = 0; + ddq->d_rtbwarns = 0; + } if (xfs_sb_version_hascrc(&mp->m_sb)) { xfs_update_cksum((char *)&dqb[j], @@ -1613,8 +1621,8 @@ xfs_qm_dqfree_one( int xfs_qm_vop_dqalloc( struct xfs_inode *ip, - xfs_dqid_t uid, - xfs_dqid_t gid, + kuid_t uid, + kgid_t gid, prid_t prid, uint flags, struct xfs_dquot **O_udqpp, @@ -1622,6 +1630,8 @@ xfs_qm_vop_dqalloc( struct xfs_dquot **O_pdqpp) { struct xfs_mount *mp = ip->i_mount; + struct inode *inode = VFS_I(ip); + struct user_namespace *user_ns = inode->i_sb->s_user_ns; struct xfs_dquot *uq = NULL; struct xfs_dquot *gq = NULL; struct xfs_dquot *pq = NULL; @@ -1635,7 +1645,7 @@ xfs_qm_vop_dqalloc( xfs_ilock(ip, lockflags); if ((flags & XFS_QMOPT_INHERIT) && XFS_INHERIT_GID(ip)) - gid = ip->i_d.di_gid; + gid = inode->i_gid; /* * Attach the dquot(s) to this inode, doing a dquot allocation @@ -1650,7 +1660,7 @@ xfs_qm_vop_dqalloc( } if ((flags & XFS_QMOPT_UQUOTA) && XFS_IS_UQUOTA_ON(mp)) { - if (ip->i_d.di_uid != uid) { + if (!uid_eq(inode->i_uid, uid)) { /* * What we need is the dquot that has this uid, and * if we send the inode to dqget, the uid of the inode @@ -1661,7 +1671,8 @@ xfs_qm_vop_dqalloc( * holding ilock. */ xfs_iunlock(ip, lockflags); - error = xfs_qm_dqget(mp, uid, XFS_DQ_USER, true, &uq); + error = xfs_qm_dqget(mp, from_kuid(user_ns, uid), + XFS_DQ_USER, true, &uq); if (error) { ASSERT(error != -ENOENT); return error; @@ -1682,9 +1693,10 @@ xfs_qm_vop_dqalloc( } } if ((flags & XFS_QMOPT_GQUOTA) && XFS_IS_GQUOTA_ON(mp)) { - if (ip->i_d.di_gid != gid) { + if (!gid_eq(inode->i_gid, gid)) { xfs_iunlock(ip, lockflags); - error = xfs_qm_dqget(mp, gid, XFS_DQ_GROUP, true, &gq); + error = xfs_qm_dqget(mp, from_kgid(user_ns, gid), + XFS_DQ_GROUP, true, &gq); if (error) { ASSERT(error != -ENOENT); goto error_rele; @@ -1810,7 +1822,7 @@ xfs_qm_vop_chown_reserve( XFS_QMOPT_RES_RTBLKS : XFS_QMOPT_RES_REGBLKS; if (XFS_IS_UQUOTA_ON(mp) && udqp && - ip->i_d.di_uid != be32_to_cpu(udqp->q_core.d_id)) { + i_uid_read(VFS_I(ip)) != be32_to_cpu(udqp->q_core.d_id)) { udq_delblks = udqp; /* * If there are delayed allocation blocks, then we have to @@ -1823,7 +1835,7 @@ xfs_qm_vop_chown_reserve( } } if (XFS_IS_GQUOTA_ON(ip->i_mount) && gdqp && - ip->i_d.di_gid != be32_to_cpu(gdqp->q_core.d_id)) { + i_gid_read(VFS_I(ip)) != be32_to_cpu(gdqp->q_core.d_id)) { gdq_delblks = gdqp; if (delblks) { ASSERT(ip->i_gdquot); @@ -1920,14 +1932,15 @@ xfs_qm_vop_create_dqattach( if (udqp && XFS_IS_UQUOTA_ON(mp)) { ASSERT(ip->i_udquot == NULL); - ASSERT(ip->i_d.di_uid == be32_to_cpu(udqp->q_core.d_id)); + ASSERT(i_uid_read(VFS_I(ip)) == be32_to_cpu(udqp->q_core.d_id)); ip->i_udquot = xfs_qm_dqhold(udqp); xfs_trans_mod_dquot(tp, udqp, XFS_TRANS_DQ_ICOUNT, 1); } if (gdqp && XFS_IS_GQUOTA_ON(mp)) { ASSERT(ip->i_gdquot == NULL); - ASSERT(ip->i_d.di_gid == be32_to_cpu(gdqp->q_core.d_id)); + ASSERT(i_gid_read(VFS_I(ip)) == be32_to_cpu(gdqp->q_core.d_id)); + ip->i_gdquot = xfs_qm_dqhold(gdqp); xfs_trans_mod_dquot(tp, gdqp, XFS_TRANS_DQ_ICOUNT, 1); } diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index 1ea82764bf89..5d5ac65aa1cc 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -29,8 +29,6 @@ xfs_qm_log_quotaoff( int error; struct xfs_qoff_logitem *qoffi; - *qoffstartp = NULL; - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_qm_quotaoff, 0, 0, 0, &tp); if (error) goto out; @@ -62,7 +60,7 @@ xfs_qm_log_quotaoff( STATIC int xfs_qm_log_quotaoff_end( struct xfs_mount *mp, - struct xfs_qoff_logitem *startqoff, + struct xfs_qoff_logitem **startqoff, uint flags) { struct xfs_trans *tp; @@ -73,9 +71,10 @@ xfs_qm_log_quotaoff_end( if (error) return error; - qoffi = xfs_trans_get_qoff_item(tp, startqoff, + qoffi = xfs_trans_get_qoff_item(tp, *startqoff, flags & XFS_ALL_QUOTA_ACCT); xfs_trans_log_quotaoff_item(tp, qoffi); + *startqoff = NULL; /* * We have to make sure that the transaction is secure on disk before we @@ -103,7 +102,7 @@ xfs_qm_scall_quotaoff( uint dqtype; int error; uint inactivate_flags; - struct xfs_qoff_logitem *qoffstart; + struct xfs_qoff_logitem *qoffstart = NULL; /* * No file system can have quotas enabled on disk but not in core. @@ -228,7 +227,7 @@ xfs_qm_scall_quotaoff( * So, we have QUOTAOFF start and end logitems; the start * logitem won't get overwritten until the end logitem appears... */ - error = xfs_qm_log_quotaoff_end(mp, qoffstart, flags); + error = xfs_qm_log_quotaoff_end(mp, &qoffstart, flags); if (error) { /* We're screwed now. Shutdown is the only option. */ xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); @@ -261,6 +260,8 @@ xfs_qm_scall_quotaoff( } out_unlock: + if (error && qoffstart) + xfs_qm_qoff_logitem_relse(qoffstart); mutex_unlock(&q->qi_quotaofflock); return error; } diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h index efe42ae7a2f3..aa8fc1f55fbd 100644 --- a/fs/xfs/xfs_quota.h +++ b/fs/xfs/xfs_quota.h @@ -86,7 +86,7 @@ extern int xfs_trans_reserve_quota_bydquots(struct xfs_trans *, struct xfs_mount *, struct xfs_dquot *, struct xfs_dquot *, struct xfs_dquot *, int64_t, long, uint); -extern int xfs_qm_vop_dqalloc(struct xfs_inode *, xfs_dqid_t, xfs_dqid_t, +extern int xfs_qm_vop_dqalloc(struct xfs_inode *, kuid_t, kgid_t, prid_t, uint, struct xfs_dquot **, struct xfs_dquot **, struct xfs_dquot **); extern void xfs_qm_vop_create_dqattach(struct xfs_trans *, struct xfs_inode *, @@ -109,7 +109,7 @@ extern void xfs_qm_unmount_quotas(struct xfs_mount *); #else static inline int -xfs_qm_vop_dqalloc(struct xfs_inode *ip, xfs_dqid_t uid, xfs_dqid_t gid, +xfs_qm_vop_dqalloc(struct xfs_inode *ip, kuid_t kuid, kgid_t kgid, prid_t prid, uint flags, struct xfs_dquot **udqp, struct xfs_dquot **gdqp, struct xfs_dquot **pdqp) { diff --git a/fs/xfs/xfs_stats.c b/fs/xfs/xfs_stats.c index 113883c4f202..f70f1255220b 100644 --- a/fs/xfs/xfs_stats.c +++ b/fs/xfs/xfs_stats.c @@ -57,13 +57,13 @@ int xfs_stats_format(struct xfsstats __percpu *stats, char *buf) /* Loop over all stats groups */ for (i = j = 0; i < ARRAY_SIZE(xstats); i++) { - len += snprintf(buf + len, PATH_MAX - len, "%s", + len += scnprintf(buf + len, PATH_MAX - len, "%s", xstats[i].desc); /* inner loop does each group */ for (; j < xstats[i].endpoint; j++) - len += snprintf(buf + len, PATH_MAX - len, " %u", + len += scnprintf(buf + len, PATH_MAX - len, " %u", counter_val(stats, j)); - len += snprintf(buf + len, PATH_MAX - len, "\n"); + len += scnprintf(buf + len, PATH_MAX - len, "\n"); } /* extra precision counters */ for_each_possible_cpu(i) { @@ -72,9 +72,9 @@ int xfs_stats_format(struct xfsstats __percpu *stats, char *buf) xs_read_bytes += per_cpu_ptr(stats, i)->s.xs_read_bytes; } - len += snprintf(buf + len, PATH_MAX-len, "xpc %Lu %Lu %Lu\n", + len += scnprintf(buf + len, PATH_MAX-len, "xpc %Lu %Lu %Lu\n", xs_xstrat_bytes, xs_write_bytes, xs_read_bytes); - len += snprintf(buf + len, PATH_MAX-len, "debug %u\n", + len += scnprintf(buf + len, PATH_MAX-len, "debug %u\n", #if defined(DEBUG) 1); #else diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index d762d42ed0ff..fa0fa3c70f1a 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -182,9 +182,7 @@ xfs_symlink( /* * Make sure that we have allocated dquot(s) on disk. */ - error = xfs_qm_vop_dqalloc(dp, - xfs_kuid_to_uid(current_fsuid()), - xfs_kgid_to_gid(current_fsgid()), prid, + error = xfs_qm_vop_dqalloc(dp, current_fsuid(), current_fsgid(), prid, XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, &udqp, &gdqp, &pdqp); if (error) @@ -194,7 +192,7 @@ xfs_symlink( * The symlink will fit into the inode data fork? * There can't be any attributes so we get the whole variable part. */ - if (pathlen <= XFS_LITINO(mp, dp->i_d.di_version)) + if (pathlen <= XFS_LITINO(mp)) fs_blocks = 0; else fs_blocks = xfs_symlink_blocks(mp, pathlen); diff --git a/fs/xfs/xfs_trace.c b/fs/xfs/xfs_trace.c index bc85b89f88ca..120398a37c2a 100644 --- a/fs/xfs/xfs_trace.c +++ b/fs/xfs/xfs_trace.c @@ -6,6 +6,7 @@ #include "xfs.h" #include "xfs_fs.h" #include "xfs_shared.h" +#include "xfs_bit.h" #include "xfs_format.h" #include "xfs_log_format.h" #include "xfs_trans_resv.h" @@ -27,6 +28,7 @@ #include "xfs_log_recover.h" #include "xfs_filestream.h" #include "xfs_fsmap.h" +#include "xfs_btree_staging.h" /* * We include this last to have the helpers above available for the trace diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index e242988f57fb..efc7751550d9 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -35,6 +35,12 @@ struct xfs_icreate_log; struct xfs_owner_info; struct xfs_trans_res; struct xfs_inobt_rec_incore; +union xfs_btree_ptr; + +#define XFS_ATTR_FILTER_FLAGS \ + { XFS_ATTR_ROOT, "ROOT" }, \ + { XFS_ATTR_SECURE, "SECURE" }, \ + { XFS_ATTR_INCOMPLETE, "INCOMPLETE" } DECLARE_EVENT_CLASS(xfs_attr_list_class, TP_PROTO(struct xfs_attr_list_context *ctx), @@ -45,39 +51,39 @@ DECLARE_EVENT_CLASS(xfs_attr_list_class, __field(u32, hashval) __field(u32, blkno) __field(u32, offset) - __field(void *, alist) + __field(void *, buffer) __field(int, bufsize) __field(int, count) __field(int, firstu) __field(int, dupcnt) - __field(int, flags) + __field(unsigned int, attr_filter) ), TP_fast_assign( __entry->dev = VFS_I(ctx->dp)->i_sb->s_dev; __entry->ino = ctx->dp->i_ino; - __entry->hashval = ctx->cursor->hashval; - __entry->blkno = ctx->cursor->blkno; - __entry->offset = ctx->cursor->offset; - __entry->alist = ctx->alist; + __entry->hashval = ctx->cursor.hashval; + __entry->blkno = ctx->cursor.blkno; + __entry->offset = ctx->cursor.offset; + __entry->buffer = ctx->buffer; __entry->bufsize = ctx->bufsize; __entry->count = ctx->count; __entry->firstu = ctx->firstu; - __entry->flags = ctx->flags; + __entry->attr_filter = ctx->attr_filter; ), TP_printk("dev %d:%d ino 0x%llx cursor h/b/o 0x%x/0x%x/%u dupcnt %u " - "alist %p size %u count %u firstu %u flags %d %s", + "buffer %p size %u count %u firstu %u filter %s", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->hashval, __entry->blkno, __entry->offset, __entry->dupcnt, - __entry->alist, + __entry->buffer, __entry->bufsize, __entry->count, __entry->firstu, - __entry->flags, - __print_flags(__entry->flags, "|", XFS_ATTR_FLAGS) + __print_flags(__entry->attr_filter, "|", + XFS_ATTR_FILTER_FLAGS) ) ) @@ -169,31 +175,31 @@ TRACE_EVENT(xfs_attr_list_node_descend, __field(u32, hashval) __field(u32, blkno) __field(u32, offset) - __field(void *, alist) + __field(void *, buffer) __field(int, bufsize) __field(int, count) __field(int, firstu) __field(int, dupcnt) - __field(int, flags) + __field(unsigned int, attr_filter) __field(u32, bt_hashval) __field(u32, bt_before) ), TP_fast_assign( __entry->dev = VFS_I(ctx->dp)->i_sb->s_dev; __entry->ino = ctx->dp->i_ino; - __entry->hashval = ctx->cursor->hashval; - __entry->blkno = ctx->cursor->blkno; - __entry->offset = ctx->cursor->offset; - __entry->alist = ctx->alist; + __entry->hashval = ctx->cursor.hashval; + __entry->blkno = ctx->cursor.blkno; + __entry->offset = ctx->cursor.offset; + __entry->buffer = ctx->buffer; __entry->bufsize = ctx->bufsize; __entry->count = ctx->count; __entry->firstu = ctx->firstu; - __entry->flags = ctx->flags; + __entry->attr_filter = ctx->attr_filter; __entry->bt_hashval = be32_to_cpu(btree->hashval); __entry->bt_before = be32_to_cpu(btree->before); ), TP_printk("dev %d:%d ino 0x%llx cursor h/b/o 0x%x/0x%x/%u dupcnt %u " - "alist %p size %u count %u firstu %u flags %d %s " + "buffer %p size %u count %u firstu %u filter %s " "node hashval %u, node before %u", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, @@ -201,12 +207,12 @@ TRACE_EVENT(xfs_attr_list_node_descend, __entry->blkno, __entry->offset, __entry->dupcnt, - __entry->alist, + __entry->buffer, __entry->bufsize, __entry->count, __entry->firstu, - __entry->flags, - __print_flags(__entry->flags, "|", XFS_ATTR_FLAGS), + __print_flags(__entry->attr_filter, "|", + XFS_ATTR_FILTER_FLAGS), __entry->bt_hashval, __entry->bt_before) ); @@ -1701,7 +1707,8 @@ DECLARE_EVENT_CLASS(xfs_attr_class, __field(int, namelen) __field(int, valuelen) __field(xfs_dahash_t, hashval) - __field(int, flags) + __field(unsigned int, attr_filter) + __field(unsigned int, attr_flags) __field(int, op_flags) ), TP_fast_assign( @@ -1712,11 +1719,12 @@ DECLARE_EVENT_CLASS(xfs_attr_class, __entry->namelen = args->namelen; __entry->valuelen = args->valuelen; __entry->hashval = args->hashval; - __entry->flags = args->flags; + __entry->attr_filter = args->attr_filter; + __entry->attr_flags = args->attr_flags; __entry->op_flags = args->op_flags; ), TP_printk("dev %d:%d ino 0x%llx name %.*s namelen %d valuelen %d " - "hashval 0x%x flags %s op_flags %s", + "hashval 0x%x filter %s flags %s op_flags %s", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->namelen, @@ -1724,7 +1732,11 @@ DECLARE_EVENT_CLASS(xfs_attr_class, __entry->namelen, __entry->valuelen, __entry->hashval, - __print_flags(__entry->flags, "|", XFS_ATTR_FLAGS), + __print_flags(__entry->attr_filter, "|", + XFS_ATTR_FILTER_FLAGS), + __print_flags(__entry->attr_flags, "|", + { XATTR_CREATE, "CREATE" }, + { XATTR_REPLACE, "REPLACE" }), __print_flags(__entry->op_flags, "|", XFS_DA_OP_FLAGS)) ) @@ -3594,6 +3606,151 @@ TRACE_EVENT(xfs_check_new_dalign, __entry->calc_rootino) ) +TRACE_EVENT(xfs_btree_commit_afakeroot, + TP_PROTO(struct xfs_btree_cur *cur), + TP_ARGS(cur), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_btnum_t, btnum) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(unsigned int, levels) + __field(unsigned int, blocks) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->btnum = cur->bc_btnum; + __entry->agno = cur->bc_ag.agno; + __entry->agbno = cur->bc_ag.afake->af_root; + __entry->levels = cur->bc_ag.afake->af_levels; + __entry->blocks = cur->bc_ag.afake->af_blocks; + ), + TP_printk("dev %d:%d btree %s ag %u levels %u blocks %u root %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->btnum, XFS_BTNUM_STRINGS), + __entry->agno, + __entry->levels, + __entry->blocks, + __entry->agbno) +) + +TRACE_EVENT(xfs_btree_commit_ifakeroot, + TP_PROTO(struct xfs_btree_cur *cur), + TP_ARGS(cur), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_btnum_t, btnum) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(unsigned int, levels) + __field(unsigned int, blocks) + __field(int, whichfork) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->btnum = cur->bc_btnum; + __entry->agno = XFS_INO_TO_AGNO(cur->bc_mp, + cur->bc_ino.ip->i_ino); + __entry->agino = XFS_INO_TO_AGINO(cur->bc_mp, + cur->bc_ino.ip->i_ino); + __entry->levels = cur->bc_ino.ifake->if_levels; + __entry->blocks = cur->bc_ino.ifake->if_blocks; + __entry->whichfork = cur->bc_ino.whichfork; + ), + TP_printk("dev %d:%d btree %s ag %u agino %u whichfork %s levels %u blocks %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->btnum, XFS_BTNUM_STRINGS), + __entry->agno, + __entry->agino, + __entry->whichfork == XFS_ATTR_FORK ? "attr" : "data", + __entry->levels, + __entry->blocks) +) + +TRACE_EVENT(xfs_btree_bload_level_geometry, + TP_PROTO(struct xfs_btree_cur *cur, unsigned int level, + uint64_t nr_this_level, unsigned int nr_per_block, + unsigned int desired_npb, uint64_t blocks, + uint64_t blocks_with_extra), + TP_ARGS(cur, level, nr_this_level, nr_per_block, desired_npb, blocks, + blocks_with_extra), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_btnum_t, btnum) + __field(unsigned int, level) + __field(unsigned int, nlevels) + __field(uint64_t, nr_this_level) + __field(unsigned int, nr_per_block) + __field(unsigned int, desired_npb) + __field(unsigned long long, blocks) + __field(unsigned long long, blocks_with_extra) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->btnum = cur->bc_btnum; + __entry->level = level; + __entry->nlevels = cur->bc_nlevels; + __entry->nr_this_level = nr_this_level; + __entry->nr_per_block = nr_per_block; + __entry->desired_npb = desired_npb; + __entry->blocks = blocks; + __entry->blocks_with_extra = blocks_with_extra; + ), + TP_printk("dev %d:%d btree %s level %u/%u nr_this_level %llu nr_per_block %u desired_npb %u blocks %llu blocks_with_extra %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->btnum, XFS_BTNUM_STRINGS), + __entry->level, + __entry->nlevels, + __entry->nr_this_level, + __entry->nr_per_block, + __entry->desired_npb, + __entry->blocks, + __entry->blocks_with_extra) +) + +TRACE_EVENT(xfs_btree_bload_block, + TP_PROTO(struct xfs_btree_cur *cur, unsigned int level, + uint64_t block_idx, uint64_t nr_blocks, + union xfs_btree_ptr *ptr, unsigned int nr_records), + TP_ARGS(cur, level, block_idx, nr_blocks, ptr, nr_records), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_btnum_t, btnum) + __field(unsigned int, level) + __field(unsigned long long, block_idx) + __field(unsigned long long, nr_blocks) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(unsigned int, nr_records) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->btnum = cur->bc_btnum; + __entry->level = level; + __entry->block_idx = block_idx; + __entry->nr_blocks = nr_blocks; + if (cur->bc_flags & XFS_BTREE_LONG_PTRS) { + xfs_fsblock_t fsb = be64_to_cpu(ptr->l); + + __entry->agno = XFS_FSB_TO_AGNO(cur->bc_mp, fsb); + __entry->agbno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsb); + } else { + __entry->agno = cur->bc_ag.agno; + __entry->agbno = be32_to_cpu(ptr->s); + } + __entry->nr_records = nr_records; + ), + TP_printk("dev %d:%d btree %s level %u block %llu/%llu fsb (%u/%u) recs %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->btnum, XFS_BTNUM_STRINGS), + __entry->level, + __entry->block_idx, + __entry->nr_blocks, + __entry->agno, + __entry->agbno, + __entry->nr_records) +) + #endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 3b208f9a865c..1adc6bc53a56 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -306,6 +306,11 @@ xfs_trans_alloc( * * Note the zero-length reservation; this transaction MUST be cancelled * without any dirty data. + * + * Callers should obtain freeze protection to avoid two conflicts with fs + * freezing: (1) having active transactions trip the m_active_trans ASSERTs; + * and (2) grabbing buffers at the same time that freeze is trying to drain + * the buffer LRU list. */ int xfs_trans_alloc_empty( @@ -450,7 +455,7 @@ xfs_trans_apply_sb_deltas( int whole = 0; bp = xfs_trans_getsb(tp, tp->t_mountp); - sbp = XFS_BUF_TO_SBP(bp); + sbp = bp->b_addr; /* * Check that superblock mods match the mods made to AGF counters. diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index 00cc5b8734be..2ef0dfbfb303 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -32,6 +32,7 @@ STATIC void xfs_ail_check( struct xfs_ail *ailp, struct xfs_log_item *lip) + __must_hold(&ailp->ail_lock) { struct xfs_log_item *prev_lip; struct xfs_log_item *next_lip; @@ -529,8 +530,9 @@ xfsaild( { struct xfs_ail *ailp = data; long tout = 0; /* milliseconds */ + unsigned int noreclaim_flag; - current->flags |= PF_MEMALLOC; + noreclaim_flag = memalloc_noreclaim_save(); set_freezable(); while (1) { @@ -601,6 +603,7 @@ xfsaild( tout = xfsaild_push(ailp); } + memalloc_noreclaim_restore(noreclaim_flag); return 0; } diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index 14473c386850..0bb82c8c6f22 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -12,9 +12,10 @@ #include "xfs_inode.h" #include "xfs_attr.h" #include "xfs_acl.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" #include -#include static int @@ -22,44 +23,20 @@ xfs_xattr_get(const struct xattr_handler *handler, struct dentry *unused, struct inode *inode, const char *name, void *value, size_t size, int flags) { - int xflags = handler->flags; - struct xfs_inode *ip = XFS_I(inode); - int error, asize = size; - size_t namelen = strlen(name); + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = handler->flags, + .name = name, + .namelen = strlen(name), + .value = value, + .valuelen = size, + }; + int error; - /* Convert Linux syscall to XFS internal ATTR flags */ - if (!size) { - xflags |= ATTR_KERNOVAL; - value = NULL; - } - - error = xfs_attr_get(ip, name, namelen, (unsigned char **)&value, - &asize, xflags); + error = xfs_attr_get(&args); if (error) return error; - return asize; -} - -void -xfs_forget_acl( - struct inode *inode, - const char *name, - int xflags) -{ - /* - * Invalidate any cached ACLs if the user has bypassed the ACL - * interface. We don't validate the content whatsoever so it is caller - * responsibility to provide data in valid format and ensure i_mode is - * consistent. - */ - if (xflags & ATTR_ROOT) { -#ifdef CONFIG_XFS_POSIX_ACL - if (!strcmp(name, SGI_ACL_FILE)) - forget_cached_acl(inode, ACL_TYPE_ACCESS); - else if (!strcmp(name, SGI_ACL_DEFAULT)) - forget_cached_acl(inode, ACL_TYPE_DEFAULT); -#endif - } + return args.valuelen; } static int @@ -67,25 +44,20 @@ xfs_xattr_set(const struct xattr_handler *handler, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) { - int xflags = handler->flags; - struct xfs_inode *ip = XFS_I(inode); + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = handler->flags, + .attr_flags = flags, + .name = name, + .namelen = strlen(name), + .value = (void *)value, + .valuelen = size, + }; int error; - size_t namelen = strlen(name); - - /* Convert Linux syscall to XFS internal ATTR flags */ - if (flags & XATTR_CREATE) - xflags |= ATTR_CREATE; - if (flags & XATTR_REPLACE) - xflags |= ATTR_REPLACE; - - if (value) - error = xfs_attr_set(ip, name, namelen, (void *)value, size, - xflags); - else - error = xfs_attr_remove(ip, name, namelen, xflags); - if (!error) - xfs_forget_acl(inode, name, xflags); + error = xfs_attr_set(&args); + if (!error && (handler->flags & XFS_ATTR_ROOT)) + xfs_forget_acl(inode, name); return error; } @@ -98,14 +70,14 @@ static const struct xattr_handler xfs_xattr_user_handler = { static const struct xattr_handler xfs_xattr_trusted_handler = { .prefix = XATTR_TRUSTED_PREFIX, - .flags = ATTR_ROOT, + .flags = XFS_ATTR_ROOT, .get = xfs_xattr_get, .set = xfs_xattr_set, }; static const struct xattr_handler xfs_xattr_security_handler = { .prefix = XATTR_SECURITY_PREFIX, - .flags = ATTR_SECURE, + .flags = XFS_ATTR_SECURE, .get = xfs_xattr_get, .set = xfs_xattr_set, }; @@ -135,7 +107,7 @@ __xfs_xattr_put_listent( if (context->count < 0 || context->seen_enough) return; - if (!context->alist) + if (!context->buffer) goto compute_size; arraytop = context->count + prefix_len + namelen + 1; @@ -144,7 +116,7 @@ __xfs_xattr_put_listent( context->seen_enough = 1; return; } - offset = (char *)context->alist + context->count; + offset = context->buffer + context->count; strncpy(offset, prefix, prefix_len); offset += prefix_len; strncpy(offset, (char *)name, namelen); /* real name */ @@ -219,7 +191,6 @@ xfs_vn_listxattr( size_t size) { struct xfs_attr_list_context context; - struct attrlist_cursor_kern cursor = { 0 }; struct inode *inode = d_inode(dentry); int error; @@ -228,14 +199,13 @@ xfs_vn_listxattr( */ memset(&context, 0, sizeof(context)); context.dp = XFS_I(inode); - context.cursor = &cursor; context.resynch = 1; - context.alist = size ? data : NULL; + context.buffer = size ? data : NULL; context.bufsize = size; context.firstu = context.bufsize; context.put_listent = xfs_xattr_put_listent; - error = xfs_attr_list_int(&context); + error = xfs_attr_list(&context); if (error) return error; if (context.count < 0) diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index cd17d50697cc..36341dfded70 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -4,6 +4,58 @@ # (This file is not included when SRCARCH=um since UML borrows several # asm headers from the host architecutre.) +mandatory-y += atomic.h +mandatory-y += barrier.h +mandatory-y += bitops.h +mandatory-y += bug.h +mandatory-y += bugs.h +mandatory-y += cacheflush.h +mandatory-y += checksum.h +mandatory-y += compat.h +mandatory-y += current.h +mandatory-y += delay.h +mandatory-y += device.h +mandatory-y += div64.h mandatory-y += dma-contiguous.h +mandatory-y += dma-mapping.h +mandatory-y += dma.h +mandatory-y += emergency-restart.h +mandatory-y += exec.h +mandatory-y += fb.h +mandatory-y += ftrace.h +mandatory-y += futex.h +mandatory-y += hardirq.h +mandatory-y += hw_irq.h +mandatory-y += io.h +mandatory-y += irq.h +mandatory-y += irq_regs.h +mandatory-y += irq_work.h +mandatory-y += kdebug.h +mandatory-y += kmap_types.h +mandatory-y += kprobes.h +mandatory-y += linkage.h +mandatory-y += local.h +mandatory-y += mm-arch-hooks.h +mandatory-y += mmiowb.h +mandatory-y += mmu.h +mandatory-y += mmu_context.h +mandatory-y += module.h mandatory-y += msi.h +mandatory-y += pci.h +mandatory-y += percpu.h +mandatory-y += pgalloc.h +mandatory-y += preempt.h +mandatory-y += sections.h +mandatory-y += serial.h +mandatory-y += shmparam.h mandatory-y += simd.h +mandatory-y += switch_to.h +mandatory-y += timex.h +mandatory-y += tlbflush.h +mandatory-y += topology.h +mandatory-y += trace_clock.h +mandatory-y += uaccess.h +mandatory-y += unaligned.h +mandatory-y += vga.h +mandatory-y += word-at-a-time.h +mandatory-y += xor.h diff --git a/include/dt-bindings/sound/meson-aiu.h b/include/dt-bindings/sound/meson-aiu.h new file mode 100644 index 000000000000..1051b8af298b --- /dev/null +++ b/include/dt-bindings/sound/meson-aiu.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __DT_MESON_AIU_H +#define __DT_MESON_AIU_H + +#define AIU_CPU 0 +#define AIU_HDMI 1 +#define AIU_ACODEC 2 + +#define CPU_I2S_FIFO 0 +#define CPU_SPDIF_FIFO 1 +#define CPU_I2S_ENCODER 2 +#define CPU_SPDIF_ENCODER 3 + +#define CTRL_I2S 0 +#define CTRL_PCM 1 +#define CTRL_OUT 2 + +#endif /* __DT_MESON_AIU_H */ diff --git a/include/dt-bindings/sound/meson-g12a-toacodec.h b/include/dt-bindings/sound/meson-g12a-toacodec.h new file mode 100644 index 000000000000..69d7a75592a2 --- /dev/null +++ b/include/dt-bindings/sound/meson-g12a-toacodec.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __DT_MESON_G12A_TOACODEC_H +#define __DT_MESON_G12A_TOACODEC_H + +#define TOACODEC_IN_A 0 +#define TOACODEC_IN_B 1 +#define TOACODEC_IN_C 2 +#define TOACODEC_OUT 3 + +#endif /* __DT_MESON_G12A_TOACODEC_H */ diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 63457908c9c4..69f4164d6477 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -231,6 +231,9 @@ struct vgic_dist { /* distributor enabled */ bool enabled; + /* Wants SGIs without active state */ + bool nassgireq; + struct vgic_irq *spis; struct vgic_io_device dist_iodev; diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index e52ceb1a73d3..99058eb81042 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -50,7 +50,13 @@ * bitmap_set(dst, pos, nbits) Set specified bit area * bitmap_clear(dst, pos, nbits) Clear specified bit area * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area - * bitmap_find_next_zero_area_off(buf, len, pos, n, mask) as above + * bitmap_find_next_zero_area_off(buf, len, pos, n, mask, mask_off) as above + * bitmap_next_clear_region(map, &start, &end, nbits) Find next clear region + * bitmap_next_set_region(map, &start, &end, nbits) Find next set region + * bitmap_for_each_clear_region(map, rs, re, start, end) + * Iterate over all clear regions + * bitmap_for_each_set_region(map, rs, re, start, end) + * Iterate over all set regions * bitmap_shift_right(dst, src, n, nbits) *dst = *src >> n * bitmap_shift_left(dst, src, n, nbits) *dst = *src << n * bitmap_cut(dst, src, first, n, nbits) Cut n bits from first, copy rest diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 63097cb243cb..e1fafed22db1 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -94,6 +94,11 @@ enum { * Enable legacy local memory.events. */ CGRP_ROOT_MEMORY_LOCAL_EVENTS = (1 << 5), + + /* + * Enable recursive subtree protection + */ + CGRP_ROOT_MEMORY_RECURSIVE_PROT = (1 << 6), }; /* cftype->flags */ diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 64461fc64e1b..21065c04c4ac 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -300,6 +300,8 @@ struct dma_router { * @chan_id: channel ID for sysfs * @dev: class device for sysfs * @name: backlink name for sysfs + * @dbg_client_name: slave name for debugfs in format: + * dev_name(requester's dev):channel name, for example: "2b00000.mcasp:tx" * @device_node: used to add this to the device chan list * @local: per-cpu pointer to a struct dma_chan_percpu * @client_count: how many clients are using this channel @@ -318,6 +320,9 @@ struct dma_chan { int chan_id; struct dma_chan_dev *dev; const char *name; +#ifdef CONFIG_DEBUG_FS + char *dbg_client_name; +#endif struct list_head device_node; struct dma_chan_percpu __percpu *local; @@ -618,10 +623,11 @@ static inline void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap) static inline void dma_descriptor_unmap(struct dma_async_tx_descriptor *tx) { - if (tx->unmap) { - dmaengine_unmap_put(tx->unmap); - tx->unmap = NULL; - } + if (!tx->unmap) + return; + + dmaengine_unmap_put(tx->unmap); + tx->unmap = NULL; } #ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH @@ -805,7 +811,9 @@ struct dma_filter { * called and there are no further references to this structure. This * must be implemented to free resources however many existing drivers * do not and are therefore not safe to unbind while in use. - * + * @dbg_summary_show: optional routine to show contents in debugfs; default code + * will be used when this is omitted, but custom code can show extra, + * controller specific information. */ struct dma_device { struct kref ref; @@ -891,6 +899,11 @@ struct dma_device { struct dma_tx_state *txstate); void (*device_issue_pending)(struct dma_chan *chan); void (*device_release)(struct dma_device *dev); + /* debugfs support */ +#ifdef CONFIG_DEBUG_FS + void (*dbg_summary_show)(struct seq_file *s, struct dma_device *dev); + struct dentry *dbg_dev_root; +#endif }; static inline int dmaengine_slave_config(struct dma_chan *chan, @@ -1155,14 +1168,7 @@ static inline dma_cookie_t dmaengine_submit(struct dma_async_tx_descriptor *desc static inline bool dmaengine_check_align(enum dmaengine_alignment align, size_t off1, size_t off2, size_t len) { - size_t mask; - - if (!align) - return true; - mask = (1 << align) - 1; - if (mask & (off1 | off2 | len)) - return false; - return true; + return !(((1 << align) - 1) & (off1 | off2 | len)); } static inline bool is_dma_copy_aligned(struct dma_device *dev, size_t off1, @@ -1236,9 +1242,9 @@ static inline int dma_maxpq(struct dma_device *dma, enum dma_ctrl_flags flags) { if (dma_dev_has_pq_continue(dma) || !dmaf_continue(flags)) return dma_dev_to_maxpq(dma); - else if (dmaf_p_disabled_continue(flags)) + if (dmaf_p_disabled_continue(flags)) return dma_dev_to_maxpq(dma) - 1; - else if (dmaf_continue(flags)) + if (dmaf_continue(flags)) return dma_dev_to_maxpq(dma) - 3; BUG(); } @@ -1249,7 +1255,7 @@ static inline size_t dmaengine_get_icg(bool inc, bool sgl, size_t icg, if (inc) { if (dir_icg) return dir_icg; - else if (sgl) + if (sgl) return icg; } @@ -1415,11 +1421,12 @@ static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie, static inline void dma_set_tx_state(struct dma_tx_state *st, dma_cookie_t last, dma_cookie_t used, u32 residue) { - if (st) { - st->last = last; - st->used = used; - st->residue = residue; - } + if (!st) + return; + + st->last = last; + st->used = used; + st->residue = residue; } #ifdef CONFIG_DMA_ENGINE @@ -1496,12 +1503,11 @@ static inline int dmaengine_desc_set_reuse(struct dma_async_tx_descriptor *tx) if (ret) return ret; - if (caps.descriptor_reuse) { - tx->flags |= DMA_CTRL_REUSE; - return 0; - } else { + if (!caps.descriptor_reuse) return -EPERM; - } + + tx->flags |= DMA_CTRL_REUSE; + return 0; } static inline void dmaengine_desc_clear_reuse(struct dma_async_tx_descriptor *tx) @@ -1517,10 +1523,10 @@ static inline bool dmaengine_desc_test_reuse(struct dma_async_tx_descriptor *tx) static inline int dmaengine_desc_free(struct dma_async_tx_descriptor *desc) { /* this is supported for reusable desc, so check that */ - if (dmaengine_desc_test_reuse(desc)) - return desc->desc_free(desc); - else + if (!dmaengine_desc_test_reuse(desc)) return -EPERM; + + return desc->desc_free(desc); } /* --- DMA device --- */ @@ -1566,9 +1572,7 @@ dmaengine_get_direction_text(enum dma_transfer_direction dir) case DMA_DEV_TO_DEV: return "DEV_TO_DEV"; default: - break; + return "invalid"; } - - return "invalid"; } #endif /* DMAENGINE_H */ diff --git a/include/linux/fs.h b/include/linux/fs.h index fbf4e1fee92d..2d140fa11dce 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -526,6 +526,11 @@ static inline void i_mmap_lock_write(struct address_space *mapping) down_write(&mapping->i_mmap_rwsem); } +static inline int i_mmap_trylock_write(struct address_space *mapping) +{ + return down_write_trylock(&mapping->i_mmap_rwsem); +} + static inline void i_mmap_unlock_write(struct address_space *mapping) { up_write(&mapping->i_mmap_rwsem); diff --git a/include/linux/gfp.h b/include/linux/gfp.h index e5b817cb86e7..be2754841369 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -485,6 +485,12 @@ static inline void arch_free_page(struct page *page, int order) { } #ifndef HAVE_ARCH_ALLOC_PAGE static inline void arch_alloc_page(struct page *page, int order) { } #endif +#ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE +static inline int arch_make_page_accessible(struct page *page) +{ + return 0; +} +#endif struct page * __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 5aca3d1bdb32..680a0d9a9721 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -87,8 +87,6 @@ extern struct kobj_attribute shmem_enabled_attr; #define HPAGE_PUD_SIZE ((1UL) << HPAGE_PUD_SHIFT) #define HPAGE_PUD_MASK (~(HPAGE_PUD_SIZE - 1)) -extern bool is_vma_temporary_stack(struct vm_area_struct *vma); - extern unsigned long transparent_hugepage_flags; /* @@ -100,7 +98,7 @@ static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma) if (vma->vm_flags & VM_NOHUGEPAGE) return false; - if (is_vma_temporary_stack(vma)) + if (vma_is_temporary_stack(vma)) return false; if (test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)) @@ -289,7 +287,11 @@ static inline struct list_head *page_deferred_list(struct page *page) #define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; }) #define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; }) -#define hpage_nr_pages(x) 1 +static inline int hpage_nr_pages(struct page *page) +{ + VM_BUG_ON_PAGE(PageTail(page), page); + return 1; +} static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma) { diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 1e897e4168ac..5ea05879a0a9 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -46,7 +46,52 @@ struct resv_map { long adds_in_progress; struct list_head region_cache; long region_cache_count; +#ifdef CONFIG_CGROUP_HUGETLB + /* + * On private mappings, the counter to uncharge reservations is stored + * here. If these fields are 0, then either the mapping is shared, or + * cgroup accounting is disabled for this resv_map. + */ + struct page_counter *reservation_counter; + unsigned long pages_per_hpage; + struct cgroup_subsys_state *css; +#endif }; + +/* + * Region tracking -- allows tracking of reservations and instantiated pages + * across the pages in a mapping. + * + * The region data structures are embedded into a resv_map and protected + * by a resv_map's lock. The set of regions within the resv_map represent + * reservations for huge pages, or huge pages that have already been + * instantiated within the map. The from and to elements are huge page + * indicies into the associated mapping. from indicates the starting index + * of the region. to represents the first index past the end of the region. + * + * For example, a file region structure with from == 0 and to == 4 represents + * four huge pages in a mapping. It is important to note that the to element + * represents the first element past the end of the region. This is used in + * arithmetic as 4(to) - 0(from) = 4 huge pages in the region. + * + * Interval notation of the form [from, to) will be used to indicate that + * the endpoint from is inclusive and to is exclusive. + */ +struct file_region { + struct list_head link; + long from; + long to; +#ifdef CONFIG_CGROUP_HUGETLB + /* + * On shared mappings, each reserved region appears as a struct + * file_region in resv_map. These fields hold the info needed to + * uncharge each reservation. + */ + struct page_counter *reservation_counter; + struct cgroup_subsys_state *css; +#endif +}; + extern struct resv_map *resv_map_alloc(void); void resv_map_release(struct kref *ref); @@ -109,6 +154,8 @@ u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx); pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud); +struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage); + extern int sysctl_hugetlb_shm_group; extern struct list_head huge_boot_pages; @@ -151,6 +198,12 @@ static inline unsigned long hugetlb_total_pages(void) return 0; } +static inline struct address_space *hugetlb_page_mapping_lock_write( + struct page *hpage) +{ + return NULL; +} + static inline int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) { @@ -390,7 +443,10 @@ static inline bool is_file_hugepages(struct file *file) return is_file_shm_hugepages(file); } - +static inline struct hstate *hstate_inode(struct inode *i) +{ + return HUGETLBFS_SB(i->i_sb)->hstate; +} #else /* !CONFIG_HUGETLBFS */ #define is_file_hugepages(file) false @@ -402,6 +458,10 @@ hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag, return ERR_PTR(-ENOSYS); } +static inline struct hstate *hstate_inode(struct inode *i) +{ + return NULL; +} #endif /* !CONFIG_HUGETLBFS */ #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA @@ -432,8 +492,8 @@ struct hstate { unsigned int surplus_huge_pages_node[MAX_NUMNODES]; #ifdef CONFIG_CGROUP_HUGETLB /* cgroup control files */ - struct cftype cgroup_files_dfl[5]; - struct cftype cgroup_files_legacy[5]; + struct cftype cgroup_files_dfl[7]; + struct cftype cgroup_files_legacy[9]; #endif char name[HSTATE_NAME_LEN]; }; @@ -472,11 +532,6 @@ extern unsigned int default_hstate_idx; #define default_hstate (hstates[default_hstate_idx]) -static inline struct hstate *hstate_inode(struct inode *i) -{ - return HUGETLBFS_SB(i->i_sb)->hstate; -} - static inline struct hstate *hstate_file(struct file *f) { return hstate_inode(file_inode(f)); @@ -729,11 +784,6 @@ static inline struct hstate *hstate_vma(struct vm_area_struct *vma) return NULL; } -static inline struct hstate *hstate_inode(struct inode *i) -{ - return NULL; -} - static inline struct hstate *page_hstate(struct page *page) { return NULL; diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index 063962f6dfc6..2ad6e92f124a 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -18,34 +18,96 @@ #include struct hugetlb_cgroup; +struct resv_map; +struct file_region; + /* * Minimum page order trackable by hugetlb cgroup. - * At least 3 pages are necessary for all the tracking information. + * At least 4 pages are necessary for all the tracking information. + * The second tail page (hpage[2]) is the fault usage cgroup. + * The third tail page (hpage[3]) is the reservation usage cgroup. */ #define HUGETLB_CGROUP_MIN_ORDER 2 #ifdef CONFIG_CGROUP_HUGETLB +enum hugetlb_memory_event { + HUGETLB_MAX, + HUGETLB_NR_MEMORY_EVENTS, +}; -static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page) +struct hugetlb_cgroup { + struct cgroup_subsys_state css; + + /* + * the counter to account for hugepages from hugetlb. + */ + struct page_counter hugepage[HUGE_MAX_HSTATE]; + + /* + * the counter to account for hugepage reservations from hugetlb. + */ + struct page_counter rsvd_hugepage[HUGE_MAX_HSTATE]; + + atomic_long_t events[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; + atomic_long_t events_local[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; + + /* Handle for "hugetlb.events" */ + struct cgroup_file events_file[HUGE_MAX_HSTATE]; + + /* Handle for "hugetlb.events.local" */ + struct cgroup_file events_local_file[HUGE_MAX_HSTATE]; +}; + +static inline struct hugetlb_cgroup * +__hugetlb_cgroup_from_page(struct page *page, bool rsvd) { VM_BUG_ON_PAGE(!PageHuge(page), page); if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER) return NULL; - return (struct hugetlb_cgroup *)page[2].private; + if (rsvd) + return (struct hugetlb_cgroup *)page[3].private; + else + return (struct hugetlb_cgroup *)page[2].private; } -static inline -int set_hugetlb_cgroup(struct page *page, struct hugetlb_cgroup *h_cg) +static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page) +{ + return __hugetlb_cgroup_from_page(page, false); +} + +static inline struct hugetlb_cgroup * +hugetlb_cgroup_from_page_rsvd(struct page *page) +{ + return __hugetlb_cgroup_from_page(page, true); +} + +static inline int __set_hugetlb_cgroup(struct page *page, + struct hugetlb_cgroup *h_cg, bool rsvd) { VM_BUG_ON_PAGE(!PageHuge(page), page); if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER) return -1; - page[2].private = (unsigned long)h_cg; + if (rsvd) + page[3].private = (unsigned long)h_cg; + else + page[2].private = (unsigned long)h_cg; return 0; } +static inline int set_hugetlb_cgroup(struct page *page, + struct hugetlb_cgroup *h_cg) +{ + return __set_hugetlb_cgroup(page, h_cg, false); +} + +static inline int set_hugetlb_cgroup_rsvd(struct page *page, + struct hugetlb_cgroup *h_cg) +{ + return __set_hugetlb_cgroup(page, h_cg, true); +} + static inline bool hugetlb_cgroup_disabled(void) { return !cgroup_subsys_enabled(hugetlb_cgrp_subsys); @@ -53,25 +115,67 @@ static inline bool hugetlb_cgroup_disabled(void) extern int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, struct hugetlb_cgroup **ptr); +extern int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages, + struct hugetlb_cgroup **ptr); extern void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg, struct page *page); +extern void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg, + struct page *page); extern void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, struct page *page); +extern void hugetlb_cgroup_uncharge_page_rsvd(int idx, unsigned long nr_pages, + struct page *page); + extern void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg); +extern void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg); +extern void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, + unsigned long start, + unsigned long end); + +extern void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, + struct file_region *rg, + unsigned long nr_pages); + extern void hugetlb_cgroup_file_init(void) __init; extern void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage); #else +static inline void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, + struct file_region *rg, + unsigned long nr_pages) +{ +} + static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page) { return NULL; } -static inline -int set_hugetlb_cgroup(struct page *page, struct hugetlb_cgroup *h_cg) +static inline struct hugetlb_cgroup * +hugetlb_cgroup_from_page_resv(struct page *page) +{ + return NULL; +} + +static inline struct hugetlb_cgroup * +hugetlb_cgroup_from_page_rsvd(struct page *page) +{ + return NULL; +} + +static inline int set_hugetlb_cgroup(struct page *page, + struct hugetlb_cgroup *h_cg) +{ + return 0; +} + +static inline int set_hugetlb_cgroup_rsvd(struct page *page, + struct hugetlb_cgroup *h_cg) { return 0; } @@ -81,28 +185,57 @@ static inline bool hugetlb_cgroup_disabled(void) return true; } -static inline int -hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, - struct hugetlb_cgroup **ptr) +static inline int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, + struct hugetlb_cgroup **ptr) { return 0; } -static inline void -hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, - struct hugetlb_cgroup *h_cg, - struct page *page) +static inline int hugetlb_cgroup_charge_cgroup_rsvd(int idx, + unsigned long nr_pages, + struct hugetlb_cgroup **ptr) +{ + return 0; +} + +static inline void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg, + struct page *page) { } static inline void -hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, struct page *page) +hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg, + struct page *page) +{ +} + +static inline void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, + struct page *page) +{ +} + +static inline void hugetlb_cgroup_uncharge_page_rsvd(int idx, + unsigned long nr_pages, + struct page *page) +{ +} +static inline void hugetlb_cgroup_uncharge_cgroup(int idx, + unsigned long nr_pages, + struct hugetlb_cgroup *h_cg) { } static inline void -hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, - struct hugetlb_cgroup *h_cg) +hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg) +{ +} + +static inline void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, + unsigned long start, + unsigned long end) { } diff --git a/include/linux/i2c-smbus.h b/include/linux/i2c-smbus.h index 585ad6fc3847..8c5459034f92 100644 --- a/include/linux/i2c-smbus.h +++ b/include/linux/i2c-smbus.h @@ -15,24 +15,19 @@ /** * i2c_smbus_alert_setup - platform data for the smbus_alert i2c client - * @alert_edge_triggered: whether the alert interrupt is edge (1) or level (0) - * triggered * @irq: IRQ number, if the smbus_alert driver should take care of interrupt * handling * * If irq is not specified, the smbus_alert driver doesn't take care of * interrupt handling. In that case it is up to the I2C bus driver to either * handle the interrupts or to poll for alerts. - * - * If irq is specified then it it crucial that alert_edge_triggered is - * properly set. */ struct i2c_smbus_alert_setup { int irq; }; -struct i2c_client *i2c_setup_smbus_alert(struct i2c_adapter *adapter, - struct i2c_smbus_alert_setup *setup); +struct i2c_client *i2c_new_smbus_alert_device(struct i2c_adapter *adapter, + struct i2c_smbus_alert_setup *setup); int i2c_handle_smbus_alert(struct i2c_client *ara); #if IS_ENABLED(CONFIG_I2C_SMBUS) && IS_ENABLED(CONFIG_OF) diff --git a/include/linux/i2c.h b/include/linux/i2c.h index f6b942150631..456fc17ecb1c 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -39,6 +39,14 @@ enum i2c_slave_event; typedef int (*i2c_slave_cb_t)(struct i2c_client *client, enum i2c_slave_event event, u8 *val); +/* I2C Frequency Modes */ +#define I2C_MAX_STANDARD_MODE_FREQ 100000 +#define I2C_MAX_FAST_MODE_FREQ 400000 +#define I2C_MAX_FAST_MODE_PLUS_FREQ 1000000 +#define I2C_MAX_TURBO_MODE_FREQ 1400000 +#define I2C_MAX_HIGH_SPEED_MODE_FREQ 3400000 +#define I2C_MAX_ULTRA_FAST_MODE_FREQ 5000000 + struct module; struct property_entry; diff --git a/include/linux/ima.h b/include/linux/ima.h index 1659217e9b60..aefe758f4466 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -30,8 +30,7 @@ extern void ima_kexec_cmdline(const void *buf, int size); extern void ima_add_kexec_buffer(struct kimage *image); #endif -#if (defined(CONFIG_X86) && defined(CONFIG_EFI)) || defined(CONFIG_S390) \ - || defined(CONFIG_PPC_SECURE_BOOT) +#ifdef CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT extern bool arch_ima_get_secureboot(void); extern const char * const *arch_get_ima_policy(void); #else diff --git a/include/linux/kasan.h b/include/linux/kasan.h index 5cde9e7c2664..31314ca7c635 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -190,7 +190,7 @@ void kasan_init_tags(void); void *kasan_reset_tag(const void *addr); -void kasan_report(unsigned long addr, size_t size, +bool kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip); #else /* CONFIG_KASAN_SW_TAGS */ diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 0f9da966934e..8bbcaad7ef0f 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -165,7 +165,8 @@ extern void __kthread_init_worker(struct kthread_worker *worker, do { \ kthread_init_work(&(dwork)->work, (fn)); \ timer_setup(&(dwork)->timer, \ - kthread_delayed_work_timer_fn, 0); \ + kthread_delayed_work_timer_fn, \ + TIMER_IRQSAFE); \ } while (0) int kthread_worker_fn(void *worker_ptr); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index bcb9b2ac0791..6d58beb65454 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -360,6 +360,10 @@ static inline unsigned long *kvm_second_dirty_bitmap(struct kvm_memory_slot *mem return memslot->dirty_bitmap + len / sizeof(*memslot->dirty_bitmap); } +#ifndef KVM_DIRTY_LOG_MANUAL_CAPS +#define KVM_DIRTY_LOG_MANUAL_CAPS KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE +#endif + struct kvm_s390_adapter_int { u64 ind_addr; u64 summary_addr; @@ -431,11 +435,11 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) */ struct kvm_memslots { u64 generation; - struct kvm_memory_slot memslots[KVM_MEM_SLOTS_NUM]; /* The mapping table from slot id to the index in memslots[]. */ short id_to_index[KVM_MEM_SLOTS_NUM]; atomic_t lru_slot; int used_slots; + struct kvm_memory_slot memslots[]; }; struct kvm { @@ -493,7 +497,7 @@ struct kvm { #endif long tlbs_dirty; struct list_head devices; - bool manual_dirty_log_protect; + u64 manual_dirty_log_protect; struct dentry *debugfs_dentry; struct kvm_stat_data **debugfs_stat_data; struct srcu_struct srcu; @@ -527,6 +531,11 @@ struct kvm { #define vcpu_err(vcpu, fmt, ...) \ kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) +static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm) +{ + return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET); +} + static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) { return srcu_dereference_check(kvm->buses[idx], &kvm->srcu, @@ -572,10 +581,11 @@ static inline int kvm_vcpu_get_idx(struct kvm_vcpu *vcpu) return vcpu->vcpu_idx; } -#define kvm_for_each_memslot(memslot, slots) \ - for (memslot = &slots->memslots[0]; \ - memslot < slots->memslots + KVM_MEM_SLOTS_NUM && memslot->npages;\ - memslot++) +#define kvm_for_each_memslot(memslot, slots) \ + for (memslot = &slots->memslots[0]; \ + memslot < slots->memslots + slots->used_slots; memslot++) \ + if (WARN_ON_ONCE(!memslot->npages)) { \ + } else void kvm_vcpu_destroy(struct kvm_vcpu *vcpu); @@ -635,12 +645,15 @@ static inline struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu) return __kvm_memslots(vcpu->kvm, as_id); } -static inline struct kvm_memory_slot * -id_to_memslot(struct kvm_memslots *slots, int id) +static inline +struct kvm_memory_slot *id_to_memslot(struct kvm_memslots *slots, int id) { int index = slots->id_to_index[id]; struct kvm_memory_slot *slot; + if (index < 0) + return NULL; + slot = &slots->memslots[index]; WARN_ON(slot->id != id); @@ -669,10 +682,7 @@ int kvm_set_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem); int __kvm_set_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem); -void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, - struct kvm_memory_slot *dont); -int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned long npages); +void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot); void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen); int kvm_arch_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *memslot, @@ -680,11 +690,9 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, enum kvm_mr_change change); void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change); -bool kvm_largepages_enabled(void); -void kvm_disable_largepages(void); /* flush all memory translations */ void kvm_arch_flush_shadow_all(struct kvm *kvm); /* flush memory translations pointing to 'slot' */ @@ -704,7 +712,6 @@ void kvm_release_page_clean(struct page *page); void kvm_release_page_dirty(struct page *page); void kvm_set_page_accessed(struct page *page); -kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn); kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn); kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, bool *writable); @@ -819,23 +826,20 @@ vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf); int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext); -int kvm_get_dirty_log(struct kvm *kvm, - struct kvm_dirty_log *log, int *is_dirty); - -int kvm_get_dirty_log_protect(struct kvm *kvm, - struct kvm_dirty_log *log, bool *flush); -int kvm_clear_dirty_log_protect(struct kvm *kvm, - struct kvm_clear_dirty_log *log, bool *flush); - void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask); +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot); -int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, - struct kvm_dirty_log *log); -int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, - struct kvm_clear_dirty_log *log); +#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot); +#else /* !CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log); +int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, + int *is_dirty, struct kvm_memory_slot **memslot); +#endif int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, bool line_status); @@ -882,9 +886,9 @@ void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu); int kvm_arch_hardware_enable(void); void kvm_arch_hardware_disable(void); -int kvm_arch_hardware_setup(void); +int kvm_arch_hardware_setup(void *opaque); void kvm_arch_hardware_unsetup(void); -int kvm_arch_check_processor_compat(void); +int kvm_arch_check_processor_compat(void *opaque); int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu); int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu); @@ -1018,6 +1022,8 @@ bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args); * used in non-modular code in arch/powerpc/kvm/book3s_hv_rm_mmu.c. * gfn_to_memslot() itself isn't here as an inline because that would * bloat other code too much. + * + * IMPORTANT: Slots are sorted from highest GFN to lowest GFN! */ static inline struct kvm_memory_slot * search_memslots(struct kvm_memslots *slots, gfn_t gfn) @@ -1026,6 +1032,9 @@ search_memslots(struct kvm_memslots *slots, gfn_t gfn) int slot = atomic_read(&slots->lru_slot); struct kvm_memory_slot *memslots = slots->memslots; + if (unlikely(!slots->used_slots)) + return NULL; + if (gfn >= memslots[slot].base_gfn && gfn < memslots[slot].base_gfn + memslots[slot].npages) return &memslots[slot]; diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index e9ba01336d4e..1b4150ff64be 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1367,12 +1367,11 @@ struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep); void memcg_kmem_put_cache(struct kmem_cache *cachep); #ifdef CONFIG_MEMCG_KMEM -int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order); -void __memcg_kmem_uncharge(struct page *page, int order); -int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, - struct mem_cgroup *memcg); -void __memcg_kmem_uncharge_memcg(struct mem_cgroup *memcg, - unsigned int nr_pages); +int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp, + unsigned int nr_pages); +void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages); +int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order); +void __memcg_kmem_uncharge_page(struct page *page, int order); extern struct static_key_false memcg_kmem_enabled_key; extern struct workqueue_struct *memcg_kmem_cache_wq; @@ -1394,32 +1393,33 @@ static inline bool memcg_kmem_enabled(void) return static_branch_unlikely(&memcg_kmem_enabled_key); } -static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order) +static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp, + int order) { if (memcg_kmem_enabled()) - return __memcg_kmem_charge(page, gfp, order); + return __memcg_kmem_charge_page(page, gfp, order); return 0; } -static inline void memcg_kmem_uncharge(struct page *page, int order) +static inline void memcg_kmem_uncharge_page(struct page *page, int order) { if (memcg_kmem_enabled()) - __memcg_kmem_uncharge(page, order); + __memcg_kmem_uncharge_page(page, order); } -static inline int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, - int order, struct mem_cgroup *memcg) +static inline int memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp, + unsigned int nr_pages) { if (memcg_kmem_enabled()) - return __memcg_kmem_charge_memcg(page, gfp, order, memcg); + return __memcg_kmem_charge(memcg, gfp, nr_pages); return 0; } -static inline void memcg_kmem_uncharge_memcg(struct page *page, int order, - struct mem_cgroup *memcg) +static inline void memcg_kmem_uncharge(struct mem_cgroup *memcg, + unsigned int nr_pages) { if (memcg_kmem_enabled()) - __memcg_kmem_uncharge_memcg(memcg, 1 << order); + __memcg_kmem_uncharge(memcg, nr_pages); } /* @@ -1436,21 +1436,23 @@ struct mem_cgroup *mem_cgroup_from_obj(void *p); #else -static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order) +static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp, + int order) { return 0; } -static inline void memcg_kmem_uncharge(struct page *page, int order) +static inline void memcg_kmem_uncharge_page(struct page *page, int order) { } -static inline int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order) +static inline int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, + int order) { return 0; } -static inline void __memcg_kmem_uncharge(struct page *page, int order) +static inline void __memcg_kmem_uncharge_page(struct page *page, int order) { } diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 5228c62af416..8165278c348a 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -173,34 +173,7 @@ extern int mpol_parse_str(char *str, struct mempolicy **mpol); extern void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol); /* Check if a vma is migratable */ -static inline bool vma_migratable(struct vm_area_struct *vma) -{ - if (vma->vm_flags & (VM_IO | VM_PFNMAP)) - return false; - - /* - * DAX device mappings require predictable access latency, so avoid - * incurring periodic faults. - */ - if (vma_is_dax(vma)) - return false; - -#ifndef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION - if (vma->vm_flags & VM_HUGETLB) - return false; -#endif - - /* - * Migration allocates pages in the highest zone. If we cannot - * do so then migration (at least from node to node) is not - * possible. - */ - if (vma->vm_file && - gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping)) - < policy_zone) - return false; - return true; -} +extern bool vma_migratable(struct vm_area_struct *vma); extern int mpol_misplaced(struct page *, struct vm_area_struct *, unsigned long); extern void mpol_put_task_policy(struct task_struct *); diff --git a/include/linux/mm.h b/include/linux/mm.h index 78c7af632c91..7f146b4bc00d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -27,6 +27,7 @@ #include #include #include +#include struct mempolicy; struct anon_vma; @@ -356,10 +357,12 @@ extern unsigned int kobjsize(const void *objp); /* * Special vmas that are non-mergable, non-mlock()able. - * Note: mm/huge_memory.c VM_NO_THP depends on this definition. */ #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP) +/* This mask prevents VMA from being scanned with khugepaged */ +#define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB) + /* This mask defines which mm->def_flags a process can inherit its parent */ #define VM_INIT_DEF_MASK VM_NOHUGEPAGE @@ -378,15 +381,75 @@ extern unsigned int kobjsize(const void *objp); */ extern pgprot_t protection_map[16]; -#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */ -#define FAULT_FLAG_MKWRITE 0x02 /* Fault was mkwrite of existing pte */ -#define FAULT_FLAG_ALLOW_RETRY 0x04 /* Retry fault if blocking */ -#define FAULT_FLAG_RETRY_NOWAIT 0x08 /* Don't drop mmap_sem and wait when retrying */ -#define FAULT_FLAG_KILLABLE 0x10 /* The fault task is in SIGKILL killable region */ -#define FAULT_FLAG_TRIED 0x20 /* Second try */ -#define FAULT_FLAG_USER 0x40 /* The fault originated in userspace */ -#define FAULT_FLAG_REMOTE 0x80 /* faulting for non current tsk/mm */ -#define FAULT_FLAG_INSTRUCTION 0x100 /* The fault was during an instruction fetch */ +/** + * Fault flag definitions. + * + * @FAULT_FLAG_WRITE: Fault was a write fault. + * @FAULT_FLAG_MKWRITE: Fault was mkwrite of existing PTE. + * @FAULT_FLAG_ALLOW_RETRY: Allow to retry the fault if blocked. + * @FAULT_FLAG_RETRY_NOWAIT: Don't drop mmap_sem and wait when retrying. + * @FAULT_FLAG_KILLABLE: The fault task is in SIGKILL killable region. + * @FAULT_FLAG_TRIED: The fault has been tried once. + * @FAULT_FLAG_USER: The fault originated in userspace. + * @FAULT_FLAG_REMOTE: The fault is not for current task/mm. + * @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch. + * @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals. + * + * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify + * whether we would allow page faults to retry by specifying these two + * fault flags correctly. Currently there can be three legal combinations: + * + * (a) ALLOW_RETRY and !TRIED: this means the page fault allows retry, and + * this is the first try + * + * (b) ALLOW_RETRY and TRIED: this means the page fault allows retry, and + * we've already tried at least once + * + * (c) !ALLOW_RETRY and !TRIED: this means the page fault does not allow retry + * + * The unlisted combination (!ALLOW_RETRY && TRIED) is illegal and should never + * be used. Note that page faults can be allowed to retry for multiple times, + * in which case we'll have an initial fault with flags (a) then later on + * continuous faults with flags (b). We should always try to detect pending + * signals before a retry to make sure the continuous page faults can still be + * interrupted if necessary. + */ +#define FAULT_FLAG_WRITE 0x01 +#define FAULT_FLAG_MKWRITE 0x02 +#define FAULT_FLAG_ALLOW_RETRY 0x04 +#define FAULT_FLAG_RETRY_NOWAIT 0x08 +#define FAULT_FLAG_KILLABLE 0x10 +#define FAULT_FLAG_TRIED 0x20 +#define FAULT_FLAG_USER 0x40 +#define FAULT_FLAG_REMOTE 0x80 +#define FAULT_FLAG_INSTRUCTION 0x100 +#define FAULT_FLAG_INTERRUPTIBLE 0x200 + +/* + * The default fault flags that should be used by most of the + * arch-specific page fault handlers. + */ +#define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \ + FAULT_FLAG_KILLABLE | \ + FAULT_FLAG_INTERRUPTIBLE) + +/** + * fault_flag_allow_retry_first - check ALLOW_RETRY the first time + * + * This is mostly used for places where we want to try to avoid taking + * the mmap_sem for too long a time when waiting for another condition + * to change, in which case we can try to be polite to release the + * mmap_sem in the first round to avoid potential starvation of other + * processes that would also want the mmap_sem. + * + * Return: true if the page fault allows retry and this is the first + * attempt of the fault handling; false otherwise. + */ +static inline bool fault_flag_allow_retry_first(unsigned int flags) +{ + return (flags & FAULT_FLAG_ALLOW_RETRY) && + (!(flags & FAULT_FLAG_TRIED)); +} #define FAULT_FLAG_TRACE \ { FAULT_FLAG_WRITE, "WRITE" }, \ @@ -397,7 +460,8 @@ extern pgprot_t protection_map[16]; { FAULT_FLAG_TRIED, "TRIED" }, \ { FAULT_FLAG_USER, "USER" }, \ { FAULT_FLAG_REMOTE, "REMOTE" }, \ - { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" } + { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ + { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" } /* * vm_fault is filled by the the pagefault handler and passed to the vma's @@ -541,6 +605,30 @@ static inline bool vma_is_anonymous(struct vm_area_struct *vma) return !vma->vm_ops; } +static inline bool vma_is_temporary_stack(struct vm_area_struct *vma) +{ + int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); + + if (!maybe_stack) + return false; + + if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == + VM_STACK_INCOMPLETE_SETUP) + return true; + + return false; +} + +static inline bool vma_is_foreign(struct vm_area_struct *vma) +{ + if (!current->mm) + return true; + + if (current->mm != vma->vm_mm) + return true; + + return false; +} #ifdef CONFIG_SHMEM /* * The vma_is_shmem is not inline because it is used only by slow @@ -770,6 +858,24 @@ static inline unsigned int compound_order(struct page *page) return page[1].compound_order; } +static inline bool hpage_pincount_available(struct page *page) +{ + /* + * Can the page->hpage_pinned_refcount field be used? That field is in + * the 3rd page of the compound page, so the smallest (2-page) compound + * pages cannot support it. + */ + page = compound_head(page); + return PageCompound(page) && compound_order(page) > 1; +} + +static inline int compound_pincount(struct page *page) +{ + VM_BUG_ON_PAGE(!hpage_pincount_available(page), page); + page = compound_head(page); + return atomic_read(compound_pincount_ptr(page)); +} + static inline void set_compound_order(struct page *page, unsigned int order) { page[1].compound_order = order; @@ -1001,6 +1107,8 @@ static inline void get_page(struct page *page) page_ref_inc(page); } +bool __must_check try_grab_page(struct page *page, unsigned int flags); + static inline __must_check bool try_get_page(struct page *page) { page = compound_head(page); @@ -1029,29 +1137,87 @@ static inline void put_page(struct page *page) __put_page(page); } -/** - * unpin_user_page() - release a gup-pinned page - * @page: pointer to page to be released +/* + * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload + * the page's refcount so that two separate items are tracked: the original page + * reference count, and also a new count of how many pin_user_pages() calls were + * made against the page. ("gup-pinned" is another term for the latter). * - * Pages that were pinned via pin_user_pages*() must be released via either - * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so - * that eventually such pages can be separately tracked and uniquely handled. In - * particular, interactions with RDMA and filesystems need special handling. + * With this scheme, pin_user_pages() becomes special: such pages are marked as + * distinct from normal pages. As such, the unpin_user_page() call (and its + * variants) must be used in order to release gup-pinned pages. * - * unpin_user_page() and put_page() are not interchangeable, despite this early - * implementation that makes them look the same. unpin_user_page() calls must - * be perfectly matched up with pin*() calls. + * Choice of value: + * + * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference + * counts with respect to pin_user_pages() and unpin_user_page() becomes + * simpler, due to the fact that adding an even power of two to the page + * refcount has the effect of using only the upper N bits, for the code that + * counts up using the bias value. This means that the lower bits are left for + * the exclusive use of the original code that increments and decrements by one + * (or at least, by much smaller values than the bias value). + * + * Of course, once the lower bits overflow into the upper bits (and this is + * OK, because subtraction recovers the original values), then visual inspection + * no longer suffices to directly view the separate counts. However, for normal + * applications that don't have huge page reference counts, this won't be an + * issue. + * + * Locking: the lockless algorithm described in page_cache_get_speculative() + * and page_cache_gup_pin_speculative() provides safe operation for + * get_user_pages and page_mkclean and other calls that race to set up page + * table entries. */ -static inline void unpin_user_page(struct page *page) -{ - put_page(page); -} +#define GUP_PIN_COUNTING_BIAS (1U << 10) +void unpin_user_page(struct page *page); void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, bool make_dirty); - void unpin_user_pages(struct page **pages, unsigned long npages); +/** + * page_maybe_dma_pinned() - report if a page is pinned for DMA. + * + * This function checks if a page has been pinned via a call to + * pin_user_pages*(). + * + * For non-huge pages, the return value is partially fuzzy: false is not fuzzy, + * because it means "definitely not pinned for DMA", but true means "probably + * pinned for DMA, but possibly a false positive due to having at least + * GUP_PIN_COUNTING_BIAS worth of normal page references". + * + * False positives are OK, because: a) it's unlikely for a page to get that many + * refcounts, and b) all the callers of this routine are expected to be able to + * deal gracefully with a false positive. + * + * For huge pages, the result will be exactly correct. That's because we have + * more tracking data available: the 3rd struct page in the compound page is + * used to track the pincount (instead using of the GUP_PIN_COUNTING_BIAS + * scheme). + * + * For more information, please see Documentation/vm/pin_user_pages.rst. + * + * @page: pointer to page to be queried. + * @Return: True, if it is likely that the page has been "dma-pinned". + * False, if the page is definitely not dma-pinned. + */ +static inline bool page_maybe_dma_pinned(struct page *page) +{ + if (hpage_pincount_available(page)) + return compound_pincount(page) > 0; + + /* + * page_ref_count() is signed. If that refcount overflows, then + * page_ref_count() returns a negative value, and callers will avoid + * further incrementing the refcount. + * + * Here, for that overflow case, use the signed bit to count a little + * bit higher via unsigned math, and thus still get an accurate result. + */ + return ((unsigned int)page_ref_count(compound_head(page))) >= + GUP_PIN_COUNTING_BIAS; +} + #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) #define SECTION_IN_PAGE_FLAGS #endif @@ -2365,26 +2531,7 @@ struct vm_unmapped_area_info { unsigned long align_offset; }; -extern unsigned long unmapped_area(struct vm_unmapped_area_info *info); -extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info); - -/* - * Search for an unmapped address range. - * - * We are looking for a range that: - * - does not intersect with any VMA; - * - is contained within the [low_limit, high_limit) interval; - * - is at least the desired size. - * - satisfies (begin_addr & align_mask) == (align_offset & align_mask) - */ -static inline unsigned long -vm_unmapped_area(struct vm_unmapped_area_info *info) -{ - if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) - return unmapped_area_topdown(info); - else - return unmapped_area(info); -} +extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info); /* truncate.c */ extern void truncate_inode_pages(struct address_space *, loff_t); diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 236099e6805d..fa3fd7224c11 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -137,7 +137,7 @@ struct page { }; struct { /* Second tail page of compound page */ unsigned long _compound_pad_1; /* compound_head */ - unsigned long _compound_pad_2; + atomic_t hpage_pinned_refcount; /* For both global and memcg */ struct list_head deferred_list; }; @@ -226,6 +226,11 @@ static inline atomic_t *compound_mapcount_ptr(struct page *page) return &page[1].compound_mapcount; } +static inline atomic_t *compound_pincount_ptr(struct page *page) +{ + return &page[2].hpage_pinned_refcount; +} + /* * Used for sizing the vmemmap region on some architectures */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 0a6f395abc68..a6c60e6efa68 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -246,6 +246,8 @@ enum node_stat_item { NR_DIRTIED, /* page dirtyings since bootup */ NR_WRITTEN, /* page writings since bootup */ NR_KERNEL_MISC_RECLAIMABLE, /* reclaimable non-slab kernel pages */ + NR_FOLL_PIN_ACQUIRED, /* via: pin_user_page(), gup flag: FOLL_PIN */ + NR_FOLL_PIN_RELEASED, /* pages returned via unpin_user_page() */ NR_VM_NODE_STAT_ITEMS }; @@ -1375,7 +1377,7 @@ static inline int pfn_valid(unsigned long pfn) } #endif -static inline int pfn_present(unsigned long pfn) +static inline int pfn_in_present_section(unsigned long pfn) { if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) return 0; @@ -1412,7 +1414,7 @@ void sparse_init(void); #else #define sparse_init() do {} while (0) #define sparse_index_init(_sec, _nid) do {} while (0) -#define pfn_present pfn_valid +#define pfn_in_present_section pfn_valid #define subsection_map_init(_pfn, _nr_pages) do {} while (0) #endif /* CONFIG_SPARSEMEM */ diff --git a/include/linux/mtd/mtd.h b/include/linux/mtd/mtd.h index 249e8d9bfbcd..2d1f4a61f4ac 100644 --- a/include/linux/mtd/mtd.h +++ b/include/linux/mtd/mtd.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -194,10 +195,43 @@ struct mtd_debug_info { const char *partid; }; +/** + * struct mtd_part - MTD partition specific fields + * + * @node: list node used to add an MTD partition to the parent partition list + * @offset: offset of the partition relatively to the parent offset + * @flags: original flags (before the mtdpart logic decided to tweak them based + * on flash constraints, like eraseblock/pagesize alignment) + * + * This struct is embedded in mtd_info and contains partition-specific + * properties/fields. + */ +struct mtd_part { + struct list_head node; + u64 offset; + u32 flags; +}; + +/** + * struct mtd_master - MTD master specific fields + * + * @partitions_lock: lock protecting accesses to the partition list. Protects + * not only the master partition list, but also all + * sub-partitions. + * @suspended: et to 1 when the device is suspended, 0 otherwise + * + * This struct is embedded in mtd_info and contains master-specific + * properties/fields. The master is the root MTD device from the MTD partition + * point of view. + */ +struct mtd_master { + struct mutex partitions_lock; + unsigned int suspended : 1; +}; + struct mtd_info { u_char type; uint32_t flags; - uint32_t orig_flags; /* Flags as before running mtd checks */ uint64_t size; // Total size of the MTD /* "Major" erase size for the device. Naïve users may take this @@ -339,8 +373,52 @@ struct mtd_info { int usecount; struct mtd_debug_info dbg; struct nvmem_device *nvmem; + + /* + * Parent device from the MTD partition point of view. + * + * MTD masters do not have any parent, MTD partitions do. The parent + * MTD device can itself be a partition. + */ + struct mtd_info *parent; + + /* List of partitions attached to this MTD device */ + struct list_head partitions; + + union { + struct mtd_part part; + struct mtd_master master; + }; }; +static inline struct mtd_info *mtd_get_master(struct mtd_info *mtd) +{ + while (mtd->parent) + mtd = mtd->parent; + + return mtd; +} + +static inline u64 mtd_get_master_ofs(struct mtd_info *mtd, u64 ofs) +{ + while (mtd->parent) { + ofs += mtd->part.offset; + mtd = mtd->parent; + } + + return ofs; +} + +static inline bool mtd_is_partition(const struct mtd_info *mtd) +{ + return mtd->parent; +} + +static inline bool mtd_has_partitions(const struct mtd_info *mtd) +{ + return !list_empty(&mtd->partitions); +} + int mtd_ooblayout_ecc(struct mtd_info *mtd, int section, struct mtd_oob_region *oobecc); int mtd_ooblayout_find_eccregion(struct mtd_info *mtd, int eccbyte, @@ -392,13 +470,16 @@ static inline u32 mtd_oobavail(struct mtd_info *mtd, struct mtd_oob_ops *ops) static inline int mtd_max_bad_blocks(struct mtd_info *mtd, loff_t ofs, size_t len) { - if (!mtd->_max_bad_blocks) + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->_max_bad_blocks) return -ENOTSUPP; if (mtd->size < (len + ofs) || ofs < 0) return -EINVAL; - return mtd->_max_bad_blocks(mtd, ofs, len); + return master->_max_bad_blocks(master, mtd_get_master_ofs(mtd, ofs), + len); } int mtd_wunit_to_pairing_info(struct mtd_info *mtd, int wunit, @@ -439,8 +520,10 @@ int mtd_writev(struct mtd_info *mtd, const struct kvec *vecs, static inline void mtd_sync(struct mtd_info *mtd) { - if (mtd->_sync) - mtd->_sync(mtd); + struct mtd_info *master = mtd_get_master(mtd); + + if (master->_sync) + master->_sync(master); } int mtd_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len); @@ -452,13 +535,31 @@ int mtd_block_markbad(struct mtd_info *mtd, loff_t ofs); static inline int mtd_suspend(struct mtd_info *mtd) { - return mtd->_suspend ? mtd->_suspend(mtd) : 0; + struct mtd_info *master = mtd_get_master(mtd); + int ret; + + if (master->master.suspended) + return 0; + + ret = master->_suspend ? master->_suspend(master) : 0; + if (ret) + return ret; + + master->master.suspended = 1; + return 0; } static inline void mtd_resume(struct mtd_info *mtd) { - if (mtd->_resume) - mtd->_resume(mtd); + struct mtd_info *master = mtd_get_master(mtd); + + if (!master->master.suspended) + return; + + if (master->_resume) + master->_resume(master); + + master->master.suspended = 0; } static inline uint32_t mtd_div_by_eb(uint64_t sz, struct mtd_info *mtd) @@ -538,7 +639,9 @@ static inline loff_t mtd_wunit_to_offset(struct mtd_info *mtd, loff_t base, static inline int mtd_has_oob(const struct mtd_info *mtd) { - return mtd->_read_oob && mtd->_write_oob; + struct mtd_info *master = mtd_get_master((struct mtd_info *)mtd); + + return master->_read_oob && master->_write_oob; } static inline int mtd_type_is_nand(const struct mtd_info *mtd) @@ -548,7 +651,9 @@ static inline int mtd_type_is_nand(const struct mtd_info *mtd) static inline int mtd_can_have_bb(const struct mtd_info *mtd) { - return !!mtd->_block_isbad; + struct mtd_info *master = mtd_get_master((struct mtd_info *)mtd); + + return !!master->_block_isbad; } /* Kernel-side ioctl definitions */ diff --git a/include/linux/mtd/partitions.h b/include/linux/mtd/partitions.h index 11cb0c50cd84..e545c050d3e8 100644 --- a/include/linux/mtd/partitions.h +++ b/include/linux/mtd/partitions.h @@ -105,7 +105,6 @@ extern void deregister_mtd_parser(struct mtd_part_parser *parser); module_driver(__mtd_part_parser, register_mtd_parser, \ deregister_mtd_parser) -int mtd_is_partition(const struct mtd_info *mtd); int mtd_add_partition(struct mtd_info *master, const char *name, long long offset, long long length); int mtd_del_partition(struct mtd_info *master, int partno); diff --git a/include/linux/mtd/rawnand.h b/include/linux/mtd/rawnand.h index 4ab9bccfcde0..1e76196f9829 100644 --- a/include/linux/mtd/rawnand.h +++ b/include/linux/mtd/rawnand.h @@ -1064,6 +1064,8 @@ struct nand_legacy { * @lock: lock protecting the suspended field. Also used to * serialize accesses to the NAND device. * @suspended: set to 1 when the device is suspended, 0 when it's not. + * @suspend: [REPLACEABLE] specific NAND device suspend operation + * @resume: [REPLACEABLE] specific NAND device resume operation * @bbt: [INTERN] bad block table pointer * @bbt_td: [REPLACEABLE] bad block table descriptor for flash * lookup. @@ -1077,6 +1079,8 @@ struct nand_legacy { * @manufacturer: [INTERN] Contains manufacturer information * @manufacturer.desc: [INTERN] Contains manufacturer's description * @manufacturer.priv: [INTERN] Contains manufacturer private information + * @lock_area: [REPLACEABLE] specific NAND chip lock operation + * @unlock_area: [REPLACEABLE] specific NAND chip unlock operation */ struct nand_chip { @@ -1117,6 +1121,8 @@ struct nand_chip { struct mutex lock; unsigned int suspended : 1; + int (*suspend)(struct nand_chip *chip); + void (*resume)(struct nand_chip *chip); uint8_t *oob_poi; struct nand_controller *controller; @@ -1136,6 +1142,9 @@ struct nand_chip { const struct nand_manufacturer *desc; void *priv; } manufacturer; + + int (*lock_area)(struct nand_chip *chip, loff_t ofs, uint64_t len); + int (*unlock_area)(struct nand_chip *chip, loff_t ofs, uint64_t len); }; extern const struct mtd_ooblayout_ops nand_ooblayout_sp_ops; @@ -1215,7 +1224,7 @@ static inline struct device_node *nand_get_flash_node(struct nand_chip *chip) * struct nand_flash_dev - NAND Flash Device ID Structure * @name: a human-readable name of the NAND chip * @dev_id: the device ID (the second byte of the full chip ID array) - * @mfr_id: manufecturer ID part of the full chip ID array (refers the same + * @mfr_id: manufacturer ID part of the full chip ID array (refers the same * memory address as ``id[0]``) * @dev_id: device ID part of the full chip ID array (refers the same memory * address as ``id[1]``) diff --git a/include/linux/mtd/spi-nor.h b/include/linux/mtd/spi-nor.h index 5abd91cc6dfa..1e2af0ec1f03 100644 --- a/include/linux/mtd/spi-nor.h +++ b/include/linux/mtd/spi-nor.h @@ -11,23 +11,6 @@ #include #include -/* - * Manufacturer IDs - * - * The first byte returned from the flash after sending opcode SPINOR_OP_RDID. - * Sometimes these are the same as CFI IDs, but sometimes they aren't. - */ -#define SNOR_MFR_ATMEL CFI_MFR_ATMEL -#define SNOR_MFR_GIGADEVICE 0xc8 -#define SNOR_MFR_INTEL CFI_MFR_INTEL -#define SNOR_MFR_ST CFI_MFR_ST /* ST Micro */ -#define SNOR_MFR_MICRON CFI_MFR_MICRON /* Micron */ -#define SNOR_MFR_ISSI CFI_MFR_PMC -#define SNOR_MFR_MACRONIX CFI_MFR_MACRONIX -#define SNOR_MFR_SPANSION CFI_MFR_AMD -#define SNOR_MFR_SST CFI_MFR_SST -#define SNOR_MFR_WINBOND 0xef /* Also used by some Spansion */ - /* * Note on opcode nomenclature: some opcodes have a format like * SPINOR_OP_FUNCTION{4,}_x_y_z. The numbers x, y, and z stand for the number @@ -128,7 +111,9 @@ #define SR_BP0 BIT(2) /* Block protect 0 */ #define SR_BP1 BIT(3) /* Block protect 1 */ #define SR_BP2 BIT(4) /* Block protect 2 */ +#define SR_BP3 BIT(5) /* Block protect 3 */ #define SR_TB_BIT5 BIT(5) /* Top/Bottom protect */ +#define SR_BP3_BIT6 BIT(6) /* Block protect 3 */ #define SR_TB_BIT6 BIT(6) /* Top/Bottom protect */ #define SR_SRWD BIT(7) /* SR write protect */ /* Spansion/Cypress specific status bits */ @@ -137,6 +122,8 @@ #define SR1_QUAD_EN_BIT6 BIT(6) +#define SR_BP_SHIFT 2 + /* Enhanced Volatile Configuration Register bits */ #define EVCR_QUAD_EN_MICRON BIT(7) /* Micron Quad I/O */ @@ -225,110 +212,6 @@ static inline u8 spi_nor_get_protocol_width(enum spi_nor_protocol proto) return spi_nor_get_protocol_data_nbits(proto); } -enum spi_nor_option_flags { - SNOR_F_USE_FSR = BIT(0), - SNOR_F_HAS_SR_TB = BIT(1), - SNOR_F_NO_OP_CHIP_ERASE = BIT(2), - SNOR_F_READY_XSR_RDY = BIT(3), - SNOR_F_USE_CLSR = BIT(4), - SNOR_F_BROKEN_RESET = BIT(5), - SNOR_F_4B_OPCODES = BIT(6), - SNOR_F_HAS_4BAIT = BIT(7), - SNOR_F_HAS_LOCK = BIT(8), - SNOR_F_HAS_16BIT_SR = BIT(9), - SNOR_F_NO_READ_CR = BIT(10), - SNOR_F_HAS_SR_TB_BIT6 = BIT(11), - -}; - -/** - * struct spi_nor_erase_type - Structure to describe a SPI NOR erase type - * @size: the size of the sector/block erased by the erase type. - * JEDEC JESD216B imposes erase sizes to be a power of 2. - * @size_shift: @size is a power of 2, the shift is stored in - * @size_shift. - * @size_mask: the size mask based on @size_shift. - * @opcode: the SPI command op code to erase the sector/block. - * @idx: Erase Type index as sorted in the Basic Flash Parameter - * Table. It will be used to synchronize the supported - * Erase Types with the ones identified in the SFDP - * optional tables. - */ -struct spi_nor_erase_type { - u32 size; - u32 size_shift; - u32 size_mask; - u8 opcode; - u8 idx; -}; - -/** - * struct spi_nor_erase_command - Used for non-uniform erases - * The structure is used to describe a list of erase commands to be executed - * once we validate that the erase can be performed. The elements in the list - * are run-length encoded. - * @list: for inclusion into the list of erase commands. - * @count: how many times the same erase command should be - * consecutively used. - * @size: the size of the sector/block erased by the command. - * @opcode: the SPI command op code to erase the sector/block. - */ -struct spi_nor_erase_command { - struct list_head list; - u32 count; - u32 size; - u8 opcode; -}; - -/** - * struct spi_nor_erase_region - Structure to describe a SPI NOR erase region - * @offset: the offset in the data array of erase region start. - * LSB bits are used as a bitmask encoding flags to - * determine if this region is overlaid, if this region is - * the last in the SPI NOR flash memory and to indicate - * all the supported erase commands inside this region. - * The erase types are sorted in ascending order with the - * smallest Erase Type size being at BIT(0). - * @size: the size of the region in bytes. - */ -struct spi_nor_erase_region { - u64 offset; - u64 size; -}; - -#define SNOR_ERASE_TYPE_MAX 4 -#define SNOR_ERASE_TYPE_MASK GENMASK_ULL(SNOR_ERASE_TYPE_MAX - 1, 0) - -#define SNOR_LAST_REGION BIT(4) -#define SNOR_OVERLAID_REGION BIT(5) - -#define SNOR_ERASE_FLAGS_MAX 6 -#define SNOR_ERASE_FLAGS_MASK GENMASK_ULL(SNOR_ERASE_FLAGS_MAX - 1, 0) - -/** - * struct spi_nor_erase_map - Structure to describe the SPI NOR erase map - * @regions: array of erase regions. The regions are consecutive in - * address space. Walking through the regions is done - * incrementally. - * @uniform_region: a pre-allocated erase region for SPI NOR with a uniform - * sector size (legacy implementation). - * @erase_type: an array of erase types shared by all the regions. - * The erase types are sorted in ascending order, with the - * smallest Erase Type size being the first member in the - * erase_type array. - * @uniform_erase_type: bitmask encoding erase types that can erase the - * entire memory. This member is completed at init by - * uniform and non-uniform SPI NOR flash memories if they - * support at least one erase type that can erase the - * entire memory. - */ -struct spi_nor_erase_map { - struct spi_nor_erase_region *regions; - struct spi_nor_erase_region uniform_region; - struct spi_nor_erase_type erase_type[SNOR_ERASE_TYPE_MAX]; - u8 uniform_erase_type; -}; - /** * struct spi_nor_hwcaps - Structure for describing the hardware capabilies * supported by the SPI controller (bus master). @@ -404,61 +287,7 @@ struct spi_nor_hwcaps { #define SNOR_HWCAPS_ALL (SNOR_HWCAPS_READ_MASK | \ SNOR_HWCAPS_PP_MASK) -struct spi_nor_read_command { - u8 num_mode_clocks; - u8 num_wait_states; - u8 opcode; - enum spi_nor_protocol proto; -}; - -struct spi_nor_pp_command { - u8 opcode; - enum spi_nor_protocol proto; -}; - -enum spi_nor_read_command_index { - SNOR_CMD_READ, - SNOR_CMD_READ_FAST, - SNOR_CMD_READ_1_1_1_DTR, - - /* Dual SPI */ - SNOR_CMD_READ_1_1_2, - SNOR_CMD_READ_1_2_2, - SNOR_CMD_READ_2_2_2, - SNOR_CMD_READ_1_2_2_DTR, - - /* Quad SPI */ - SNOR_CMD_READ_1_1_4, - SNOR_CMD_READ_1_4_4, - SNOR_CMD_READ_4_4_4, - SNOR_CMD_READ_1_4_4_DTR, - - /* Octal SPI */ - SNOR_CMD_READ_1_1_8, - SNOR_CMD_READ_1_8_8, - SNOR_CMD_READ_8_8_8, - SNOR_CMD_READ_1_8_8_DTR, - - SNOR_CMD_READ_MAX -}; - -enum spi_nor_pp_command_index { - SNOR_CMD_PP, - - /* Quad SPI */ - SNOR_CMD_PP_1_1_4, - SNOR_CMD_PP_1_4_4, - SNOR_CMD_PP_4_4_4, - - /* Octal SPI */ - SNOR_CMD_PP_1_1_8, - SNOR_CMD_PP_1_8_8, - SNOR_CMD_PP_8_8_8, - - SNOR_CMD_PP_MAX -}; - -/* Forward declaration that will be used in 'struct spi_nor_flash_parameter' */ +/* Forward declaration that is used in 'struct spi_nor_controller_ops' */ struct spi_nor; /** @@ -489,68 +318,13 @@ struct spi_nor_controller_ops { int (*erase)(struct spi_nor *nor, loff_t offs); }; -/** - * struct spi_nor_locking_ops - SPI NOR locking methods - * @lock: lock a region of the SPI NOR. - * @unlock: unlock a region of the SPI NOR. - * @is_locked: check if a region of the SPI NOR is completely locked - */ -struct spi_nor_locking_ops { - int (*lock)(struct spi_nor *nor, loff_t ofs, uint64_t len); - int (*unlock)(struct spi_nor *nor, loff_t ofs, uint64_t len); - int (*is_locked)(struct spi_nor *nor, loff_t ofs, uint64_t len); -}; - -/** - * struct spi_nor_flash_parameter - SPI NOR flash parameters and settings. - * Includes legacy flash parameters and settings that can be overwritten - * by the spi_nor_fixups hooks, or dynamically when parsing the JESD216 - * Serial Flash Discoverable Parameters (SFDP) tables. - * - * @size: the flash memory density in bytes. - * @page_size: the page size of the SPI NOR flash memory. - * @hwcaps: describes the read and page program hardware - * capabilities. - * @reads: read capabilities ordered by priority: the higher index - * in the array, the higher priority. - * @page_programs: page program capabilities ordered by priority: the - * higher index in the array, the higher priority. - * @erase_map: the erase map parsed from the SFDP Sector Map Parameter - * Table. - * @quad_enable: enables SPI NOR quad mode. - * @set_4byte: puts the SPI NOR in 4 byte addressing mode. - * @convert_addr: converts an absolute address into something the flash - * will understand. Particularly useful when pagesize is - * not a power-of-2. - * @setup: configures the SPI NOR memory. Useful for SPI NOR - * flashes that have peculiarities to the SPI NOR standard - * e.g. different opcodes, specific address calculation, - * page size, etc. - * @locking_ops: SPI NOR locking methods. - */ -struct spi_nor_flash_parameter { - u64 size; - u32 page_size; - - struct spi_nor_hwcaps hwcaps; - struct spi_nor_read_command reads[SNOR_CMD_READ_MAX]; - struct spi_nor_pp_command page_programs[SNOR_CMD_PP_MAX]; - - struct spi_nor_erase_map erase_map; - - int (*quad_enable)(struct spi_nor *nor); - int (*set_4byte)(struct spi_nor *nor, bool enable); - u32 (*convert_addr)(struct spi_nor *nor, u32 addr); - int (*setup)(struct spi_nor *nor, const struct spi_nor_hwcaps *hwcaps); - - const struct spi_nor_locking_ops *locking_ops; -}; - -/** - * struct flash_info - Forward declaration of a structure used internally by - * spi_nor_scan() +/* + * Forward declarations that are used internally by the core and manufacturer + * drivers. */ struct flash_info; +struct spi_nor_manufacturer; +struct spi_nor_flash_parameter; /** * struct spi_nor - Structure for defining a the SPI NOR layer @@ -562,6 +336,7 @@ struct flash_info; * layer is not DMA-able * @bouncebuf_size: size of the bounce buffer * @info: spi-nor part JDEC MFR id and other info + * @manufacturer: spi-nor manufacturer * @page_size: the page size of the SPI NOR * @addr_width: number of address bytes * @erase_opcode: the opcode for erasing a sector @@ -578,6 +353,7 @@ struct flash_info; * The structure includes legacy flash parameters and * settings that can be overwritten by the spi_nor_fixups * hooks, or dynamically when parsing the SFDP tables. + * @dirmap: pointers to struct spi_mem_dirmap_desc for reads/writes. * @priv: the private data */ struct spi_nor { @@ -588,6 +364,7 @@ struct spi_nor { u8 *bouncebuf; size_t bouncebuf_size; const struct flash_info *info; + const struct spi_nor_manufacturer *manufacturer; u32 page_size; u8 addr_width; u8 erase_opcode; @@ -602,40 +379,16 @@ struct spi_nor { const struct spi_nor_controller_ops *controller_ops; - struct spi_nor_flash_parameter params; + struct spi_nor_flash_parameter *params; + + struct { + struct spi_mem_dirmap_desc *rdesc; + struct spi_mem_dirmap_desc *wdesc; + } dirmap; void *priv; }; -static u64 __maybe_unused -spi_nor_region_is_last(const struct spi_nor_erase_region *region) -{ - return region->offset & SNOR_LAST_REGION; -} - -static u64 __maybe_unused -spi_nor_region_end(const struct spi_nor_erase_region *region) -{ - return (region->offset & ~SNOR_ERASE_FLAGS_MASK) + region->size; -} - -static void __maybe_unused -spi_nor_region_mark_end(struct spi_nor_erase_region *region) -{ - region->offset |= SNOR_LAST_REGION; -} - -static void __maybe_unused -spi_nor_region_mark_overlay(struct spi_nor_erase_region *region) -{ - region->offset |= SNOR_OVERLAID_REGION; -} - -static bool __maybe_unused spi_nor_has_uniform_erase(const struct spi_nor *nor) -{ - return !!nor->params.erase_map.uniform_erase_type; -} - static inline void spi_nor_set_flash_node(struct spi_nor *nor, struct device_node *np) { diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index 4ea558bd3c46..1077c45721ff 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -32,9 +32,9 @@ SPI_MEM_OP_NO_DUMMY, \ SPI_MEM_OP_NO_DATA) -#define SPINAND_READID_OP(ndummy, buf, len) \ +#define SPINAND_READID_OP(naddr, ndummy, buf, len) \ SPI_MEM_OP(SPI_MEM_OP_CMD(0x9f, 1), \ - SPI_MEM_OP_NO_ADDR, \ + SPI_MEM_OP_ADDR(naddr, 0, 1), \ SPI_MEM_OP_DUMMY(ndummy, 1), \ SPI_MEM_OP_DATA_IN(len, buf, 1)) @@ -176,37 +176,46 @@ struct spinand_device; * @data: buffer containing the id bytes. Currently 4 bytes large, but can * be extended if required * @len: ID length - * - * struct_spinand_id->data contains all bytes returned after a READ_ID command, - * including dummy bytes if the chip does not emit ID bytes right after the - * READ_ID command. The responsibility to extract real ID bytes is left to - * struct_manufacurer_ops->detect(). */ struct spinand_id { u8 data[SPINAND_MAX_ID_LEN]; int len; }; +enum spinand_readid_method { + SPINAND_READID_METHOD_OPCODE, + SPINAND_READID_METHOD_OPCODE_ADDR, + SPINAND_READID_METHOD_OPCODE_DUMMY, +}; + +/** + * struct spinand_devid - SPI NAND device id structure + * @id: device id of current chip + * @len: number of bytes in device id + * @method: method to read chip id + * There are 3 possible variants: + * SPINAND_READID_METHOD_OPCODE: chip id is returned immediately + * after read_id opcode. + * SPINAND_READID_METHOD_OPCODE_ADDR: chip id is returned after + * read_id opcode + 1-byte address. + * SPINAND_READID_METHOD_OPCODE_DUMMY: chip id is returned after + * read_id opcode + 1 dummy byte. + */ +struct spinand_devid { + const u8 *id; + const u8 len; + const enum spinand_readid_method method; +}; + /** * struct manufacurer_ops - SPI NAND manufacturer specific operations - * @detect: detect a SPI NAND device. Every time a SPI NAND device is probed - * the core calls the struct_manufacurer_ops->detect() hook of each - * registered manufacturer until one of them return 1. Note that - * the first thing to check in this hook is that the manufacturer ID - * in struct_spinand_device->id matches the manufacturer whose - * ->detect() hook has been called. Should return 1 if there's a - * match, 0 if the manufacturer ID does not match and a negative - * error code otherwise. When true is returned, the core assumes - * that properties of the NAND chip (spinand->base.memorg and - * spinand->base.eccreq) have been filled * @init: initialize a SPI NAND device * @cleanup: cleanup a SPI NAND device * * Each SPI NAND manufacturer driver should implement this interface so that - * NAND chips coming from this vendor can be detected and initialized properly. + * NAND chips coming from this vendor can be initialized properly. */ struct spinand_manufacturer_ops { - int (*detect)(struct spinand_device *spinand); int (*init)(struct spinand_device *spinand); void (*cleanup)(struct spinand_device *spinand); }; @@ -215,11 +224,16 @@ struct spinand_manufacturer_ops { * struct spinand_manufacturer - SPI NAND manufacturer instance * @id: manufacturer ID * @name: manufacturer name + * @devid_len: number of bytes in device ID + * @chips: supported SPI NANDs under current manufacturer + * @nchips: number of SPI NANDs available in chips array * @ops: manufacturer operations */ struct spinand_manufacturer { u8 id; char *name; + const struct spinand_info *chips; + const size_t nchips; const struct spinand_manufacturer_ops *ops; }; @@ -270,6 +284,7 @@ struct spinand_ecc_info { }; #define SPINAND_HAS_QE_BIT BIT(0) +#define SPINAND_HAS_CR_FEAT_BIT BIT(1) /** * struct spinand_info - Structure used to describe SPI NAND chips @@ -291,7 +306,7 @@ struct spinand_ecc_info { */ struct spinand_info { const char *model; - u16 devid; + struct spinand_devid devid; u32 flags; struct nand_memory_organization memorg; struct nand_ecc_req eccreq; @@ -305,6 +320,13 @@ struct spinand_info { unsigned int target); }; +#define SPINAND_ID(__method, ...) \ + { \ + .id = (const u8[]){ __VA_ARGS__ }, \ + .len = sizeof((u8[]){ __VA_ARGS__ }), \ + .method = __method, \ + } + #define SPINAND_INFO_OP_VARIANTS(__read, __write, __update) \ { \ .read_cache = __read, \ @@ -451,9 +473,10 @@ static inline void spinand_set_of_node(struct spinand_device *spinand, nanddev_set_of_node(&spinand->base, np); } -int spinand_match_and_init(struct spinand_device *dev, +int spinand_match_and_init(struct spinand_device *spinand, const struct spinand_info *table, - unsigned int table_size, u16 devid); + unsigned int table_size, + enum spinand_readid_method rdid_method); int spinand_upd_cfg(struct spinand_device *spinand, u8 mask, u8 val); int spinand_select_target(struct spinand_device *spinand, unsigned int target); diff --git a/include/linux/page_ref.h b/include/linux/page_ref.h index 14d14beb1f7f..d27701199a4d 100644 --- a/include/linux/page_ref.h +++ b/include/linux/page_ref.h @@ -102,6 +102,15 @@ static inline void page_ref_sub(struct page *page, int nr) __page_ref_mod(page, -nr); } +static inline int page_ref_sub_return(struct page *page, int nr) +{ + int ret = atomic_sub_return(nr, &page->_refcount); + + if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_return)) + __page_ref_mod_and_return(page, -nr, ret); + return ret; +} + static inline void page_ref_inc(struct page *page) { atomic_inc(&page->_refcount); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index ccb14b6a16b5..f56282491a48 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -33,8 +33,8 @@ enum mapping_flags { /** * mapping_set_error - record a writeback error in the address_space - * @mapping - the mapping in which an error should be set - * @error - the error to set in the mapping + * @mapping: the mapping in which an error should be set + * @error: the error to set in the mapping * * When writeback fails in some way, we must record that error so that * userspace can be informed when fsync and the like are called. We endeavor @@ -70,11 +70,9 @@ static inline void mapping_clear_unevictable(struct address_space *mapping) clear_bit(AS_UNEVICTABLE, &mapping->flags); } -static inline int mapping_unevictable(struct address_space *mapping) +static inline bool mapping_unevictable(struct address_space *mapping) { - if (mapping) - return test_bit(AS_UNEVICTABLE, &mapping->flags); - return !!mapping; + return mapping && test_bit(AS_UNEVICTABLE, &mapping->flags); } static inline void mapping_set_exiting(struct address_space *mapping) @@ -305,9 +303,9 @@ static inline struct page *find_lock_page(struct address_space *mapping, * atomic allocation! */ static inline struct page *find_or_create_page(struct address_space *mapping, - pgoff_t offset, gfp_t gfp_mask) + pgoff_t index, gfp_t gfp_mask) { - return pagecache_get_page(mapping, offset, + return pagecache_get_page(mapping, index, FGP_LOCK|FGP_ACCESSED|FGP_CREAT, gfp_mask); } @@ -333,14 +331,19 @@ static inline struct page *grab_cache_page_nowait(struct address_space *mapping, mapping_gfp_mask(mapping)); } -static inline struct page *find_subpage(struct page *page, pgoff_t offset) +/* + * Given the page we found in the page cache, return the page corresponding + * to this index in the file + */ +static inline struct page *find_subpage(struct page *head, pgoff_t index) { - if (PageHuge(page)) - return page; + /* HugeTLBfs wants the head page regardless */ + if (PageHuge(head)) + return head; - VM_BUG_ON_PAGE(PageTail(page), page); + VM_BUG_ON_PAGE(PageTail(head), head); - return page + (offset & (compound_nr(page) - 1)); + return head + (index & (compound_nr(head) - 1)); } struct page *find_get_entry(struct address_space *mapping, pgoff_t offset); diff --git a/include/linux/platform_data/remoteproc-omap.h b/include/linux/platform_data/remoteproc-omap.h deleted file mode 100644 index 7e3a16097672..000000000000 --- a/include/linux/platform_data/remoteproc-omap.h +++ /dev/null @@ -1,51 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Remote Processor - omap-specific bits - * - * Copyright (C) 2011 Texas Instruments, Inc. - * Copyright (C) 2011 Google, Inc. - */ - -#ifndef _PLAT_REMOTEPROC_H -#define _PLAT_REMOTEPROC_H - -struct rproc_ops; -struct platform_device; - -/* - * struct omap_rproc_pdata - omap remoteproc's platform data - * @name: the remoteproc's name - * @oh_name: omap hwmod device - * @oh_name_opt: optional, secondary omap hwmod device - * @firmware: name of firmware file to load - * @mbox_name: name of omap mailbox device to use with this rproc - * @ops: start/stop rproc handlers - * @device_enable: omap-specific handler for enabling a device - * @device_shutdown: omap-specific handler for shutting down a device - * @set_bootaddr: omap-specific handler for setting the rproc boot address - */ -struct omap_rproc_pdata { - const char *name; - const char *oh_name; - const char *oh_name_opt; - const char *firmware; - const char *mbox_name; - const struct rproc_ops *ops; - int (*device_enable)(struct platform_device *pdev); - int (*device_shutdown)(struct platform_device *pdev); - void (*set_bootaddr)(u32); -}; - -#if defined(CONFIG_OMAP_REMOTEPROC) || defined(CONFIG_OMAP_REMOTEPROC_MODULE) - -void __init omap_rproc_reserve_cma(void); - -#else - -static inline void __init omap_rproc_reserve_cma(void) -{ -} - -#endif - -#endif /* _PLAT_REMOTEPROC_H */ diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index 16ad66683ad0..9c07d7958c53 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -329,7 +329,7 @@ struct rproc; struct rproc_mem_entry { void *va; dma_addr_t dma; - int len; + size_t len; u32 da; void *priv; char name[32]; @@ -369,12 +369,14 @@ enum rsc_handling_status { * expects to find it * @sanity_check: sanity check the fw image * @get_boot_addr: get boot address to entry point specified in firmware + * @panic: optional callback to react to system panic, core will delay + * panic at least the returned number of milliseconds */ struct rproc_ops { int (*start)(struct rproc *rproc); int (*stop)(struct rproc *rproc); void (*kick)(struct rproc *rproc, int vqid); - void * (*da_to_va)(struct rproc *rproc, u64 da, int len); + void * (*da_to_va)(struct rproc *rproc, u64 da, size_t len); int (*parse_fw)(struct rproc *rproc, const struct firmware *fw); int (*handle_rsc)(struct rproc *rproc, u32 rsc_type, void *rsc, int offset, int avail); @@ -382,7 +384,8 @@ struct rproc_ops { struct rproc *rproc, const struct firmware *fw); int (*load)(struct rproc *rproc, const struct firmware *fw); int (*sanity_check)(struct rproc *rproc, const struct firmware *fw); - u32 (*get_boot_addr)(struct rproc *rproc, const struct firmware *fw); + u64 (*get_boot_addr)(struct rproc *rproc, const struct firmware *fw); + unsigned long (*panic)(struct rproc *rproc); }; /** @@ -498,7 +501,7 @@ struct rproc { int num_traces; struct list_head carveouts; struct list_head mappings; - u32 bootaddr; + u64 bootaddr; struct list_head rvdevs; struct list_head subdevs; struct idr notifyids; @@ -514,6 +517,7 @@ struct rproc { bool auto_boot; struct list_head dump_segments; int nb_vdev; + u8 elf_class; }; /** @@ -599,13 +603,13 @@ void rproc_add_carveout(struct rproc *rproc, struct rproc_mem_entry *mem); struct rproc_mem_entry * rproc_mem_entry_init(struct device *dev, - void *va, dma_addr_t dma, int len, u32 da, + void *va, dma_addr_t dma, size_t len, u32 da, int (*alloc)(struct rproc *, struct rproc_mem_entry *), int (*release)(struct rproc *, struct rproc_mem_entry *), const char *name, ...); struct rproc_mem_entry * -rproc_of_resm_mem_entry_init(struct device *dev, u32 of_resm_idx, int len, +rproc_of_resm_mem_entry_init(struct device *dev, u32 of_resm_idx, size_t len, u32 da, const char *name, ...); int rproc_boot(struct rproc *rproc); diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index a29df79540ce..3e5b090c16d4 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -10,6 +10,8 @@ #include #include #include +#include +#include /* * Types defining task->signal and task->sighand and APIs using them: @@ -376,6 +378,20 @@ static inline int signal_pending_state(long state, struct task_struct *p) return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); } +/* + * This should only be used in fault handlers to decide whether we + * should stop the current fault routine to handle the signals + * instead, especially with the case where we've got interrupted with + * a VM_FAULT_RETRY. + */ +static inline bool fault_signal_pending(vm_fault_t fault_flags, + struct pt_regs *regs) +{ + return unlikely((fault_flags & VM_FAULT_RETRY) && + (fatal_signal_pending(current) || + (user_mode(regs) && signal_pending(current)))); +} + /* * Reevaluate whether the task has signals pending delivery. * Wake the task if so. diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h index b451bb622335..2dfe14ed3bb0 100644 --- a/include/linux/soundwire/sdw.h +++ b/include/linux/soundwire/sdw.h @@ -284,6 +284,7 @@ struct sdw_dpn_audio_mode { * @max_async_buffer: Number of samples that this port can buffer in * asynchronous modes * @block_pack_mode: Type of block port mode supported + * @read_only_wordlength: Read Only wordlength field in DPN_BlockCtrl1 register * @port_encoding: Payload Channel Sample encoding schemes supported * @audio_modes: Audio modes supported */ @@ -307,6 +308,7 @@ struct sdw_dpn_prop { u32 modes; u32 max_async_buffer; bool block_pack_mode; + bool read_only_wordlength; u32 port_encoding; struct sdw_dpn_audio_mode *audio_modes; }; diff --git a/include/linux/swap.h b/include/linux/swap.h index 1e99f7ac1d7e..b835d8dbea0e 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -374,7 +374,6 @@ extern int sysctl_min_slab_ratio; #define node_reclaim_mode 0 #endif -extern int page_evictable(struct page *page); extern void check_move_unevictable_pages(struct pagevec *pvec); extern int kswapd_run(int nid); diff --git a/include/linux/topology.h b/include/linux/topology.h index eb2fe6edd73c..608fa4aadf0e 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -130,20 +130,11 @@ static inline int numa_node_id(void) * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem(). */ DECLARE_PER_CPU(int, _numa_mem_); -extern int _node_numa_mem_[MAX_NUMNODES]; #ifndef set_numa_mem static inline void set_numa_mem(int node) { this_cpu_write(_numa_mem_, node); - _node_numa_mem_[numa_node_id()] = node; -} -#endif - -#ifndef node_to_mem_node -static inline int node_to_mem_node(int node) -{ - return _node_numa_mem_[node]; } #endif @@ -166,7 +157,6 @@ static inline int cpu_to_mem(int cpu) static inline void set_cpu_numa_mem(int cpu, int node) { per_cpu(_numa_mem_, cpu) = node; - _node_numa_mem_[cpu_to_node(cpu)] = node; } #endif @@ -180,13 +170,6 @@ static inline int numa_mem_id(void) } #endif -#ifndef node_to_mem_node -static inline int node_to_mem_node(int node) -{ - return node; -} -#endif - #ifndef cpu_to_mem static inline int cpu_to_mem(int cpu) { diff --git a/include/linux/usb/audio-v2.h b/include/linux/usb/audio-v2.h index 5e31740c7e40..ead8c9a47c6a 100644 --- a/include/linux/usb/audio-v2.h +++ b/include/linux/usb/audio-v2.h @@ -156,6 +156,18 @@ struct uac2_feature_unit_descriptor { __u8 bmaControls[]; /* variable length */ } __attribute__((packed)); +/* 4.7.2.10 Effect Unit Descriptor */ + +struct uac2_effect_unit_descriptor { + __u8 bLength; + __u8 bDescriptorType; + __u8 bDescriptorSubtype; + __u8 bUnitID; + __le16 wEffectType; + __u8 bSourceID; + __u8 bmaControls[]; /* variable length */ +} __attribute__((packed)); + /* 4.9.2 Class-Specific AS Interface Descriptor */ struct uac2_as_header_descriptor { diff --git a/include/sound/compress_driver.h b/include/sound/compress_driver.h index bc88d6f964da..6ce8effa0b12 100644 --- a/include/sound/compress_driver.h +++ b/include/sound/compress_driver.h @@ -23,7 +23,6 @@ struct snd_compr_ops; * struct snd_compr_runtime: runtime stream description * @state: stream state * @ops: pointer to DSP callbacks - * @dma_buffer_p: runtime dma buffer pointer * @buffer: pointer to kernel buffer, valid only when not in mmap mode or * DSP doesn't implement copy * @buffer_size: size of the above buffer @@ -34,11 +33,14 @@ struct snd_compr_ops; * @total_bytes_transferred: cumulative bytes transferred by offload DSP * @sleep: poll sleep * @private_data: driver private data pointer + * @dma_area: virtual buffer address + * @dma_addr: physical buffer address (not accessible from main CPU) + * @dma_bytes: size of DMA area + * @dma_buffer_p: runtime dma buffer pointer */ struct snd_compr_runtime { snd_pcm_state_t state; struct snd_compr_ops *ops; - struct snd_dma_buffer *dma_buffer_p; void *buffer; u64 buffer_size; u32 fragment_size; @@ -47,6 +49,11 @@ struct snd_compr_runtime { u64 total_bytes_transferred; wait_queue_head_t sleep; void *private_data; + + unsigned char *dma_area; + dma_addr_t dma_addr; + size_t dma_bytes; + struct snd_dma_buffer *dma_buffer_p; }; /** @@ -60,6 +67,7 @@ struct snd_compr_runtime { * @metadata_set: metadata set flag, true when set * @next_track: has userspace signal next track transition, true when set * @private_data: pointer to DSP private data + * @dma_buffer: allocated buffer if any */ struct snd_compr_stream { const char *name; @@ -71,6 +79,7 @@ struct snd_compr_stream { bool metadata_set; bool next_track; void *private_data; + struct snd_dma_buffer dma_buffer; }; /** @@ -180,21 +189,34 @@ static inline void snd_compr_drain_notify(struct snd_compr_stream *stream) /** * snd_compr_set_runtime_buffer - Set the Compress runtime buffer - * @substream: compress substream to set + * @stream: compress stream to set * @bufp: the buffer information, NULL to clear * * Copy the buffer information to runtime buffer when @bufp is non-NULL. * Otherwise it clears the current buffer information. */ -static inline void snd_compr_set_runtime_buffer( - struct snd_compr_stream *substream, - struct snd_dma_buffer *bufp) +static inline void +snd_compr_set_runtime_buffer(struct snd_compr_stream *stream, + struct snd_dma_buffer *bufp) { - struct snd_compr_runtime *runtime = substream->runtime; + struct snd_compr_runtime *runtime = stream->runtime; - runtime->dma_buffer_p = bufp; + if (bufp) { + runtime->dma_buffer_p = bufp; + runtime->dma_area = bufp->area; + runtime->dma_addr = bufp->addr; + runtime->dma_bytes = bufp->bytes; + } else { + runtime->dma_buffer_p = NULL; + runtime->dma_area = NULL; + runtime->dma_addr = 0; + runtime->dma_bytes = 0; + } } +int snd_compr_malloc_pages(struct snd_compr_stream *stream, size_t size); +int snd_compr_free_pages(struct snd_compr_stream *stream); + int snd_compr_stop_error(struct snd_compr_stream *stream, snd_pcm_state_t state); diff --git a/include/sound/core.h b/include/sound/core.h index ac8b692b69b4..381a010a1bd4 100644 --- a/include/sound/core.h +++ b/include/sound/core.h @@ -266,6 +266,7 @@ void snd_device_disconnect(struct snd_card *card, void *device_data); void snd_device_disconnect_all(struct snd_card *card); void snd_device_free(struct snd_card *card, void *device_data); void snd_device_free_all(struct snd_card *card); +int snd_device_get_state(struct snd_card *card, void *device_data); /* isadma.c */ diff --git a/include/sound/hdaudio.h b/include/sound/hdaudio.h index d4299e146d95..affedc2801c4 100644 --- a/include/sound/hdaudio.h +++ b/include/sound/hdaudio.h @@ -513,6 +513,7 @@ struct hdac_stream { struct snd_pcm_substream *substream; /* assigned substream, * set in PCM open */ + struct snd_compr_stream *cstream; unsigned int format_val; /* format value to be set in the * controller and the codec */ @@ -527,6 +528,7 @@ struct hdac_stream { bool locked:1; bool stripe:1; /* apply stripe control */ + u64 curr_pos; /* timestamp */ unsigned long start_wallclk; /* start + minimum wallclk */ unsigned long period_wallclk; /* wallclk for period */ diff --git a/include/sound/pcm.h b/include/sound/pcm.h index f657ff08f317..2ba5df2c9e23 100644 --- a/include/sound/pcm.h +++ b/include/sound/pcm.h @@ -644,6 +644,11 @@ void snd_pcm_stream_unlock_irqrestore(struct snd_pcm_substream *substream, #define snd_pcm_group_for_each_entry(s, substream) \ list_for_each_entry(s, &substream->group->substreams, link_list) +#define for_each_pcm_streams(stream) \ + for (stream = SNDRV_PCM_STREAM_PLAYBACK; \ + stream <= SNDRV_PCM_STREAM_LAST; \ + stream++) + /** * snd_pcm_running - Check whether the substream is in a running state * @substream: substream to check @@ -1122,7 +1127,14 @@ snd_pcm_kernel_readv(struct snd_pcm_substream *substream, return __snd_pcm_lib_xfer(substream, bufs, false, frames, true); } -int snd_pcm_limit_hw_rates(struct snd_pcm_runtime *runtime); +int snd_pcm_hw_limit_rates(struct snd_pcm_hardware *hw); + +static inline int +snd_pcm_limit_hw_rates(struct snd_pcm_runtime *runtime) +{ + return snd_pcm_hw_limit_rates(&runtime->hw); +} + unsigned int snd_pcm_rate_to_rate_bit(unsigned int rate); unsigned int snd_pcm_rate_bit_to_rate(unsigned int rate_bit); unsigned int snd_pcm_rate_mask_intersect(unsigned int rates_a, @@ -1415,6 +1427,15 @@ static inline u64 pcm_format_to_bits(snd_pcm_format_t pcm_format) return 1ULL << (__force int) pcm_format; } +/** + * pcm_for_each_format - helper to iterate for each format type + * @f: the iterator variable in snd_pcm_format_t type + */ +#define pcm_for_each_format(f) \ + for ((f) = SNDRV_PCM_FORMAT_FIRST; \ + (__force int)(f) <= (__force int)SNDRV_PCM_FORMAT_LAST; \ + (f) = (__force snd_pcm_format_t)((__force int)(f) + 1)) + /* printk helpers */ #define pcm_err(pcm, fmt, args...) \ dev_err((pcm)->card->dev, fmt, ##args) diff --git a/include/sound/pcm_params.h b/include/sound/pcm_params.h index 661450a2095b..36f94735d23d 100644 --- a/include/sound/pcm_params.h +++ b/include/sound/pcm_params.h @@ -133,6 +133,13 @@ static inline int snd_mask_test(const struct snd_mask *mask, unsigned int val) return mask->bits[MASK_OFS(val)] & MASK_BIT(val); } +/* Most of drivers need only this one */ +static inline int snd_mask_test_format(const struct snd_mask *mask, + snd_pcm_format_t format) +{ + return snd_mask_test(mask, (__force unsigned int)format); +} + static inline int snd_mask_single(const struct snd_mask *mask) { int i, c = 0; diff --git a/include/sound/rt5682.h b/include/sound/rt5682.h index bc2c31734df1..e1f790561ac1 100644 --- a/include/sound/rt5682.h +++ b/include/sound/rt5682.h @@ -24,6 +24,12 @@ enum rt5682_jd_src { RT5682_JD1, }; +enum rt5682_dai_clks { + RT5682_DAI_WCLK_IDX, + RT5682_DAI_BCLK_IDX, + RT5682_DAI_NUM_CLKS, +}; + struct rt5682_platform_data { int ldo1_en; /* GPIO for LDO1_EN */ @@ -32,6 +38,10 @@ struct rt5682_platform_data { enum rt5682_dmic1_clk_pin dmic1_clk_pin; enum rt5682_jd_src jd_src; unsigned int btndet_delay; + unsigned int dmic_clk_rate; + unsigned int dmic_delay; + + const char *dai_clk_names[RT5682_DAI_NUM_CLKS]; }; #endif diff --git a/include/sound/soc-acpi.h b/include/sound/soc-acpi.h index a217a87cae86..392e953d561e 100644 --- a/include/sound/soc-acpi.h +++ b/include/sound/soc-acpi.h @@ -75,18 +75,45 @@ struct snd_soc_acpi_mach_params { }; /** - * snd_soc_acpi_link_adr: ACPI-based list of _ADR, with a variable - * number of devices per link - * + * snd_soc_acpi_endpoint - endpoint descriptor + * @num: endpoint number (mandatory, unique per device) + * @aggregated: 0 (independent) or 1 (logically grouped) + * @group_position: zero-based order (only when @aggregated is 1) + * @group_id: platform-unique group identifier (only when @aggregrated is 1) + */ +struct snd_soc_acpi_endpoint { + u8 num; + u8 aggregated; + u8 group_position; + u8 group_id; +}; + +/** + * snd_soc_acpi_adr_device - descriptor for _ADR-enumerated device + * @adr: 64 bit ACPI _ADR value + * @num_endpoints: number of endpoints for this device + * @endpoints: array of endpoints + */ +struct snd_soc_acpi_adr_device { + const u64 adr; + const u8 num_endpoints; + const struct snd_soc_acpi_endpoint *endpoints; +}; + +/** + * snd_soc_acpi_link_adr - ACPI-based list of _ADR enumerated devices * @mask: one bit set indicates the link this list applies to - * @num_adr: ARRAY_SIZE of adr - * @adr: array of _ADR (represented as u64). + * @num_adr: ARRAY_SIZE of devices + * @adr_d: array of devices + * + * The number of devices per link can be more than 1, e.g. in SoundWire + * multi-drop configurations. */ struct snd_soc_acpi_link_adr { const u32 mask; const u32 num_adr; - const u64 *adr; + const struct snd_soc_acpi_adr_device *adr_d; }; /** diff --git a/include/sound/soc-dai.h b/include/sound/soc-dai.h index eaaeb00e9e84..78bac995db15 100644 --- a/include/sound/soc-dai.h +++ b/include/sound/soc-dai.h @@ -202,6 +202,8 @@ struct snd_soc_dai_ops { int (*set_sdw_stream)(struct snd_soc_dai *dai, void *stream, int direction); + void *(*get_sdw_stream)(struct snd_soc_dai *dai, int direction); + /* * DAI digital mute - optional. * Called by soc-core to minimise any pops. @@ -322,9 +324,7 @@ struct snd_soc_dai { struct snd_soc_dai_driver *driver; /* DAI runtime info */ - unsigned int capture_active; /* stream usage count */ - unsigned int playback_active; /* stream usage count */ - unsigned int probed:1; + unsigned int stream_active[SNDRV_PCM_STREAM_LAST + 1]; /* usage count */ unsigned int active; @@ -348,8 +348,27 @@ struct snd_soc_dai { unsigned int rx_mask; struct list_head list; + + /* bit field */ + unsigned int probed:1; + unsigned int started:1; }; +static inline struct snd_soc_pcm_stream * +snd_soc_dai_get_pcm_stream(const struct snd_soc_dai *dai, int stream) +{ + return (stream == SNDRV_PCM_STREAM_PLAYBACK) ? + &dai->driver->playback : &dai->driver->capture; +} + +static inline +struct snd_soc_dapm_widget *snd_soc_dai_get_widget( + struct snd_soc_dai *dai, int stream) +{ + return (stream == SNDRV_PCM_STREAM_PLAYBACK) ? + dai->playback_widget : dai->capture_widget; +} + static inline void *snd_soc_dai_get_dma_data(const struct snd_soc_dai *dai, const struct snd_pcm_substream *ss) { @@ -406,4 +425,23 @@ static inline int snd_soc_dai_set_sdw_stream(struct snd_soc_dai *dai, return -ENOTSUPP; } +/** + * snd_soc_dai_get_sdw_stream() - Retrieves SDW stream from DAI + * @dai: DAI + * @direction: Stream direction(Playback/Capture) + * + * This routine only retrieves that was previously configured + * with snd_soc_dai_get_sdw_stream() + * + * Returns pointer to stream or -ENOTSUPP if callback is not supported; + */ +static inline void *snd_soc_dai_get_sdw_stream(struct snd_soc_dai *dai, + int direction) +{ + if (dai->driver->ops->get_sdw_stream) + return dai->driver->ops->get_sdw_stream(dai, direction); + else + return ERR_PTR(-ENOTSUPP); +} + #endif diff --git a/include/sound/soc-dapm.h b/include/sound/soc-dapm.h index 1b6afbc1a4ed..08495f8d86dc 100644 --- a/include/sound/soc-dapm.h +++ b/include/sound/soc-dapm.h @@ -482,6 +482,7 @@ int snd_soc_dapm_dai_get_connected_widgets(struct snd_soc_dai *dai, int stream, struct snd_soc_dapm_widget_list **list, bool (*custom_stop_condition)(struct snd_soc_dapm_widget *, enum snd_soc_dapm_direction)); +void snd_soc_dapm_dai_free_widgets(struct snd_soc_dapm_widget_list **list); struct snd_soc_dapm_context *snd_soc_dapm_kcontrol_dapm( struct snd_kcontrol *kcontrol); @@ -691,6 +692,11 @@ struct snd_soc_dapm_widget_list { struct snd_soc_dapm_widget *widgets[0]; }; +#define for_each_dapm_widgets(list, i, widget) \ + for ((i) = 0; \ + (i) < list->num_widgets && (widget = list->widgets[i]); \ + (i)++) + struct snd_soc_dapm_stats { int power_checks; int path_checks; diff --git a/include/sound/soc-dpcm.h b/include/sound/soc-dpcm.h index b654ebfc8766..0f6c50b17bba 100644 --- a/include/sound/soc-dpcm.h +++ b/include/sound/soc-dpcm.h @@ -132,17 +132,8 @@ int snd_soc_dpcm_be_can_update(struct snd_soc_pcm_runtime *fe, struct snd_pcm_substream * snd_soc_dpcm_get_substream(struct snd_soc_pcm_runtime *be, int stream); -/* get the BE runtime state */ -enum snd_soc_dpcm_state - snd_soc_dpcm_be_get_state(struct snd_soc_pcm_runtime *be, int stream); - -/* set the BE runtime state */ -void snd_soc_dpcm_be_set_state(struct snd_soc_pcm_runtime *be, int stream, - enum snd_soc_dpcm_state state); - -/* internal use only */ -int soc_dpcm_be_digital_mute(struct snd_soc_pcm_runtime *fe, int mute); -int soc_dpcm_runtime_update(struct snd_soc_card *); +/* update audio routing between PCMs and any DAI links */ +int snd_soc_dpcm_runtime_update(struct snd_soc_card *card); #ifdef CONFIG_DEBUG_FS void soc_dpcm_debugfs_add(struct snd_soc_pcm_runtime *rtd); @@ -154,6 +145,7 @@ static inline void soc_dpcm_debugfs_add(struct snd_soc_pcm_runtime *rtd) int dpcm_path_get(struct snd_soc_pcm_runtime *fe, int stream, struct snd_soc_dapm_widget_list **list_); +void dpcm_path_put(struct snd_soc_dapm_widget_list **list); int dpcm_process_paths(struct snd_soc_pcm_runtime *fe, int stream, struct snd_soc_dapm_widget_list **list, int new); int dpcm_be_dai_startup(struct snd_soc_pcm_runtime *fe, int stream); @@ -167,10 +159,4 @@ int dpcm_be_dai_prepare(struct snd_soc_pcm_runtime *fe, int stream); int dpcm_dapm_stream_event(struct snd_soc_pcm_runtime *fe, int dir, int event); -static inline void dpcm_path_put(struct snd_soc_dapm_widget_list **list) -{ - kfree(*list); -} - - #endif diff --git a/include/sound/soc.h b/include/sound/soc.h index b905c7369e28..f3cb653d40bc 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -479,6 +479,9 @@ bool snd_soc_runtime_ignore_pmdown_time(struct snd_soc_pcm_runtime *rtd); void snd_soc_runtime_activate(struct snd_soc_pcm_runtime *rtd, int stream); void snd_soc_runtime_deactivate(struct snd_soc_pcm_runtime *rtd, int stream); +int snd_soc_runtime_calc_hw(struct snd_soc_pcm_runtime *rtd, + struct snd_pcm_hardware *hw, int stream); + int snd_soc_runtime_set_dai_fmt(struct snd_soc_pcm_runtime *rtd, unsigned int dai_fmt); @@ -865,6 +868,11 @@ struct snd_soc_dai_link { ((platform) = &link->platforms[i]); \ (i)++) +#define for_each_link_cpus(link, i, cpu) \ + for ((i) = 0; \ + ((i) < link->num_cpus) && ((cpu) = &link->cpus[i]); \ + (i)++) + /* * Sample 1 : Single CPU/Codec/Platform * @@ -1068,6 +1076,7 @@ struct snd_soc_card { const struct snd_soc_dapm_route *of_dapm_routes; int num_of_dapm_routes; bool fully_routed; + bool disable_route_checks; /* lists of probed devices belonging to this card */ struct list_head component_dev_list; @@ -1119,6 +1128,14 @@ struct snd_soc_card { #define for_each_card_components(card, component) \ list_for_each_entry(component, &(card)->component_dev_list, card_list) +#define for_each_card_dapms(card, dapm) \ + list_for_each_entry(dapm, &card->dapm_list, list) + +#define for_each_card_widgets(card, w)\ + list_for_each_entry(w, &card->widgets, list) +#define for_each_card_widgets_safe(card, w, _w) \ + list_for_each_entry_safe(w, _w, &card->widgets, list) + /* SoC machine DAI configuration, glues a codec and cpu DAI together */ struct snd_soc_pcm_runtime { struct device *dev; @@ -1138,10 +1155,14 @@ struct snd_soc_pcm_runtime { struct snd_compr *compr; struct snd_soc_dai *codec_dai; struct snd_soc_dai *cpu_dai; + struct snd_soc_dai **dais; struct snd_soc_dai **codec_dais; unsigned int num_codecs; + struct snd_soc_dai **cpu_dais; + unsigned int num_cpus; + struct delayed_work delayed_work; void (*close_delayed_work_func)(struct snd_soc_pcm_runtime *rtd); #ifdef CONFIG_DEBUG_FS @@ -1158,16 +1179,31 @@ struct snd_soc_pcm_runtime { int num_components; struct snd_soc_component *components[0]; /* CPU/Codec/Platform */ }; +/* see soc_new_pcm_runtime() */ +#define asoc_rtd_to_cpu(rtd, n) (rtd)->dais[n] +#define asoc_rtd_to_codec(rtd, n) (rtd)->dais[n + (rtd)->num_cpus] + #define for_each_rtd_components(rtd, i, component) \ for ((i) = 0; \ ((i) < rtd->num_components) && ((component) = rtd->components[i]);\ (i)++) -#define for_each_rtd_codec_dai(rtd, i, dai)\ - for ((i) = 0; \ - ((i) < rtd->num_codecs) && ((dai) = rtd->codec_dais[i]); \ +#define for_each_rtd_cpu_dais(rtd, i, dai) \ + for ((i) = 0; \ + ((i) < rtd->num_cpus) && ((dai) = rtd->cpu_dais[i]); \ (i)++) -#define for_each_rtd_codec_dai_rollback(rtd, i, dai) \ +#define for_each_rtd_cpu_dais_rollback(rtd, i, dai) \ + for (; (--(i) >= 0) && ((dai) = rtd->cpu_dais[i]);) +#define for_each_rtd_codec_dais(rtd, i, dai) \ + for ((i) = 0; \ + ((i) < rtd->num_codecs) && ((dai) = rtd->codec_dais[i]); \ + (i)++) +#define for_each_rtd_codec_dais_rollback(rtd, i, dai) \ for (; (--(i) >= 0) && ((dai) = rtd->codec_dais[i]);) +#define for_each_rtd_dais(rtd, i, dai) \ + for ((i) = 0; \ + ((i) < (rtd)->num_cpus + (rtd)->num_codecs) && \ + ((dai) = (rtd)->dais[i]); \ + (i)++) void snd_soc_close_delayed_work(struct snd_soc_pcm_runtime *rtd); diff --git a/include/sound/sof/dai-intel.h b/include/sound/sof/dai-intel.h index 5f1ef5565be6..04e48227f542 100644 --- a/include/sound/sof/dai-intel.h +++ b/include/sound/sof/dai-intel.h @@ -87,6 +87,15 @@ struct sof_ipc_dai_hda_params { uint32_t link_dma_ch; } __packed; +/* ALH Configuration Request - SOF_IPC_DAI_ALH_CONFIG */ +struct sof_ipc_dai_alh_params { + struct sof_ipc_hdr hdr; + uint32_t stream_id; + + /* reserved for future use */ + uint32_t reserved[15]; +} __packed; + /* DMIC Configuration Request - SOF_IPC_DAI_DMIC_CONFIG */ /* This struct is defined per 2ch PDM controller available in the platform. @@ -179,13 +188,4 @@ struct sof_ipc_dai_dmic_params { struct sof_ipc_dai_dmic_pdm_ctrl pdm[0]; } __packed; -/* ALH Configuration Request - SOF_IPC_DAI_ALH_CONFIG */ -struct sof_ipc_dai_alh_params { - struct sof_ipc_hdr hdr; - uint32_t stream_id; - - /* reserved for future use */ - uint32_t reserved[15]; -} __packed; - #endif diff --git a/include/sound/sof/header.h b/include/sound/sof/header.h index bf3edd9c08b4..b79479575cc8 100644 --- a/include/sound/sof/header.h +++ b/include/sound/sof/header.h @@ -51,6 +51,7 @@ #define SOF_IPC_GLB_TRACE_MSG SOF_GLB_TYPE(0x9U) #define SOF_IPC_GLB_GDB_DEBUG SOF_GLB_TYPE(0xAU) #define SOF_IPC_GLB_TEST_MSG SOF_GLB_TYPE(0xBU) +#define SOF_IPC_GLB_PROBE SOF_GLB_TYPE(0xCU) /* * DSP Command Message Types @@ -102,6 +103,16 @@ #define SOF_IPC_STREAM_VORBIS_PARAMS SOF_CMD_TYPE(0x010) #define SOF_IPC_STREAM_VORBIS_FREE SOF_CMD_TYPE(0x011) +/* probe */ +#define SOF_IPC_PROBE_INIT SOF_CMD_TYPE(0x001) +#define SOF_IPC_PROBE_DEINIT SOF_CMD_TYPE(0x002) +#define SOF_IPC_PROBE_DMA_ADD SOF_CMD_TYPE(0x003) +#define SOF_IPC_PROBE_DMA_INFO SOF_CMD_TYPE(0x004) +#define SOF_IPC_PROBE_DMA_REMOVE SOF_CMD_TYPE(0x005) +#define SOF_IPC_PROBE_POINT_ADD SOF_CMD_TYPE(0x006) +#define SOF_IPC_PROBE_POINT_INFO SOF_CMD_TYPE(0x007) +#define SOF_IPC_PROBE_POINT_REMOVE SOF_CMD_TYPE(0x008) + /* trace */ #define SOF_IPC_TRACE_DMA_PARAMS SOF_CMD_TYPE(0x001) #define SOF_IPC_TRACE_DMA_POSITION SOF_CMD_TYPE(0x002) diff --git a/include/sound/sof/info.h b/include/sound/sof/info.h index 1c560144996c..438a11fcf272 100644 --- a/include/sound/sof/info.h +++ b/include/sound/sof/info.h @@ -28,9 +28,9 @@ /* extended data types that can be appended onto end of sof_ipc_fw_ready */ enum sof_ipc_ext_data { - SOF_IPC_EXT_DMA_BUFFER = 0, - SOF_IPC_EXT_WINDOW, - SOF_IPC_EXT_CC_INFO, + SOF_IPC_EXT_UNUSED = 0, + SOF_IPC_EXT_WINDOW = 1, + SOF_IPC_EXT_CC_INFO = 2, }; /* FW version - SOF_IPC_GLB_VERSION */ @@ -83,22 +83,6 @@ struct sof_ipc_ext_data_hdr { uint32_t type; /**< SOF_IPC_EXT_ */ } __packed; -struct sof_ipc_dma_buffer_elem { - struct sof_ipc_hdr hdr; - uint32_t type; /**< SOF_IPC_REGION_ */ - uint32_t id; /**< platform specific - used to map to host memory */ - struct sof_ipc_host_buffer buffer; -} __packed; - -/* extended data DMA buffers for IPC, trace and debug */ -struct sof_ipc_dma_buffer_data { - struct sof_ipc_ext_data_hdr ext_hdr; - uint32_t num_buffers; - - /* host files in buffer[n].buffer */ - struct sof_ipc_dma_buffer_elem buffer[]; -} __packed; - struct sof_ipc_window_elem { struct sof_ipc_hdr hdr; uint32_t type; /**< SOF_IPC_REGION_ */ diff --git a/include/sound/sof/topology.h b/include/sound/sof/topology.h index 8e76178fedf0..402e0250c508 100644 --- a/include/sound/sof/topology.h +++ b/include/sound/sof/topology.h @@ -53,9 +53,10 @@ struct sof_ipc_comp { uint32_t id; enum sof_comp_type type; uint32_t pipeline_id; + uint32_t core; /* reserved for future use */ - uint32_t reserved[2]; + uint32_t reserved[1]; } __packed; /* diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h new file mode 100644 index 000000000000..4661f7ba07c0 --- /dev/null +++ b/include/trace/events/mmap.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mmap + +#if !defined(_TRACE_MMAP_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_MMAP_H + +#include + +TRACE_EVENT(vm_unmapped_area, + + TP_PROTO(unsigned long addr, struct vm_unmapped_area_info *info), + + TP_ARGS(addr, info), + + TP_STRUCT__entry( + __field(unsigned long, addr) + __field(unsigned long, total_vm) + __field(unsigned long, flags) + __field(unsigned long, length) + __field(unsigned long, low_limit) + __field(unsigned long, high_limit) + __field(unsigned long, align_mask) + __field(unsigned long, align_offset) + ), + + TP_fast_assign( + __entry->addr = addr; + __entry->total_vm = current->mm->total_vm; + __entry->flags = info->flags; + __entry->length = info->length; + __entry->low_limit = info->low_limit; + __entry->high_limit = info->high_limit; + __entry->align_mask = info->align_mask; + __entry->align_offset = info->align_offset; + ), + + TP_printk("addr=0x%lx err=%ld total_vm=0x%lx flags=0x%lx len=0x%lx lo=0x%lx hi=0x%lx mask=0x%lx ofs=0x%lx\n", + IS_ERR_VALUE(__entry->addr) ? 0 : __entry->addr, + IS_ERR_VALUE(__entry->addr) ? __entry->addr : 0, + __entry->total_vm, __entry->flags, __entry->length, + __entry->low_limit, __entry->high_limit, __entry->align_mask, + __entry->align_offset) +); +#endif + +/* This part must be outside protection */ +#include diff --git a/include/uapi/linux/idxd.h b/include/uapi/linux/idxd.h index 849ef1515d04..1f412fbf561b 100644 --- a/include/uapi/linux/idxd.h +++ b/include/uapi/linux/idxd.h @@ -83,21 +83,6 @@ enum dsa_completion_status { #define DSA_COMP_STATUS_MASK 0x7f #define DSA_COMP_STATUS_WRITE 0x80 -struct dsa_batch_desc { - uint32_t pasid:20; - uint32_t rsvd:11; - uint32_t priv:1; - uint32_t flags:24; - uint32_t opcode:8; - uint64_t completion_addr; - uint64_t desc_list_addr; - uint64_t rsvd1; - uint32_t desc_count; - uint16_t interrupt_handle; - uint16_t rsvd2; - uint8_t rsvd3[24]; -} __attribute__((packed)); - struct dsa_hw_desc { uint32_t pasid:20; uint32_t rsvd:11; @@ -109,6 +94,7 @@ struct dsa_hw_desc { uint64_t src_addr; uint64_t rdback_addr; uint64_t pattern; + uint64_t desc_list_addr; }; union { uint64_t dst_addr; @@ -116,7 +102,10 @@ struct dsa_hw_desc { uint64_t src2_addr; uint64_t comp_pattern; }; - uint32_t xfer_size; + union { + uint32_t xfer_size; + uint32_t desc_count; + }; uint16_t int_handle; uint16_t rsvd1; union { diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 4b95f9a31a2f..428c7dde6b4b 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -474,12 +474,17 @@ struct kvm_s390_mem_op { __u32 size; /* amount of bytes */ __u32 op; /* type of operation */ __u64 buf; /* buffer in userspace */ - __u8 ar; /* the access register number */ - __u8 reserved[31]; /* should be set to 0 */ + union { + __u8 ar; /* the access register number */ + __u32 sida_offset; /* offset into the sida */ + __u8 reserved[32]; /* should be set to 0 */ + }; }; /* types for kvm_s390_mem_op->op */ #define KVM_S390_MEMOP_LOGICAL_READ 0 #define KVM_S390_MEMOP_LOGICAL_WRITE 1 +#define KVM_S390_MEMOP_SIDA_READ 2 +#define KVM_S390_MEMOP_SIDA_WRITE 3 /* flags for kvm_s390_mem_op->flags */ #define KVM_S390_MEMOP_F_CHECK_ONLY (1ULL << 0) #define KVM_S390_MEMOP_F_INJECT_EXCEPTION (1ULL << 1) @@ -1010,6 +1015,8 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_ARM_NISV_TO_USER 177 #define KVM_CAP_ARM_INJECT_EXT_DABT 178 #define KVM_CAP_S390_VCPU_RESETS 179 +#define KVM_CAP_S390_PROTECTED 180 +#define KVM_CAP_PPC_SECURE_GUEST 181 #ifdef KVM_CAP_IRQ_ROUTING @@ -1478,6 +1485,39 @@ struct kvm_enc_region { #define KVM_S390_NORMAL_RESET _IO(KVMIO, 0xc3) #define KVM_S390_CLEAR_RESET _IO(KVMIO, 0xc4) +struct kvm_s390_pv_sec_parm { + __u64 origin; + __u64 length; +}; + +struct kvm_s390_pv_unp { + __u64 addr; + __u64 size; + __u64 tweak; +}; + +enum pv_cmd_id { + KVM_PV_ENABLE, + KVM_PV_DISABLE, + KVM_PV_SET_SEC_PARMS, + KVM_PV_UNPACK, + KVM_PV_VERIFY, + KVM_PV_PREP_RESET, + KVM_PV_UNSHARE_ALL, +}; + +struct kvm_pv_cmd { + __u32 cmd; /* Command to be executed */ + __u16 rc; /* Ultravisor return code */ + __u16 rrc; /* Ultravisor return reason code */ + __u64 data; /* Data or address */ + __u32 flags; /* flags for future extensions. Must be 0 for now */ + __u32 reserved[3]; +}; + +/* Available with KVM_CAP_S390_PROTECTED */ +#define KVM_S390_PV_COMMAND _IOWR(KVMIO, 0xc5, struct kvm_pv_cmd) + /* Secure Encrypted Virtualization command */ enum sev_cmd_id { /* Guest initialization commands */ @@ -1628,4 +1668,7 @@ struct kvm_hyperv_eventfd { #define KVM_HYPERV_CONN_ID_MASK 0x00ffffff #define KVM_HYPERV_EVENTFD_DEASSIGN (1 << 0) +#define KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE (1 << 0) +#define KVM_DIRTY_LOG_INITIALLY_SET (1 << 1) + #endif /* __LINUX_KVM_H */ diff --git a/include/uapi/linux/mman.h b/include/uapi/linux/mman.h index fc1a64c3447b..923cc162609c 100644 --- a/include/uapi/linux/mman.h +++ b/include/uapi/linux/mman.h @@ -5,8 +5,9 @@ #include #include -#define MREMAP_MAYMOVE 1 -#define MREMAP_FIXED 2 +#define MREMAP_MAYMOVE 1 +#define MREMAP_FIXED 2 +#define MREMAP_DONTUNMAP 4 #define OVERCOMMIT_GUESS 0 #define OVERCOMMIT_ALWAYS 1 diff --git a/include/uapi/sound/asoc.h b/include/uapi/sound/asoc.h index 6048553c119d..a74ca232f1fc 100644 --- a/include/uapi/sound/asoc.h +++ b/include/uapi/sound/asoc.h @@ -17,6 +17,7 @@ #define __LINUX_UAPI_SND_ASOC_H #include +#include /* * Maximum number of channels topology kcontrol can represent. diff --git a/include/uapi/sound/compress_offload.h b/include/uapi/sound/compress_offload.h index 56d95673ce0f..7184265c0b0d 100644 --- a/include/uapi/sound/compress_offload.h +++ b/include/uapi/sound/compress_offload.h @@ -31,7 +31,7 @@ #include -#define SNDRV_COMPRESS_VERSION SNDRV_PROTOCOL_VERSION(0, 1, 2) +#define SNDRV_COMPRESS_VERSION SNDRV_PROTOCOL_VERSION(0, 2, 0) /** * struct snd_compressed_buffer - compressed buffer * @fragment_size: size of buffer fragment in bytes diff --git a/include/uapi/sound/compress_params.h b/include/uapi/sound/compress_params.h index 9c96fb0e4d90..79b14389ae41 100644 --- a/include/uapi/sound/compress_params.h +++ b/include/uapi/sound/compress_params.h @@ -75,7 +75,9 @@ #define SND_AUDIOCODEC_G723_1 ((__u32) 0x0000000C) #define SND_AUDIOCODEC_G729 ((__u32) 0x0000000D) #define SND_AUDIOCODEC_BESPOKE ((__u32) 0x0000000E) -#define SND_AUDIOCODEC_MAX SND_AUDIOCODEC_BESPOKE +#define SND_AUDIOCODEC_ALAC ((__u32) 0x0000000F) +#define SND_AUDIOCODEC_APE ((__u32) 0x00000010) +#define SND_AUDIOCODEC_MAX SND_AUDIOCODEC_APE /* * Profile and modes are listed with bit masks. This allows for a @@ -142,6 +144,9 @@ #define SND_AUDIOPROFILE_WMA8 ((__u32) 0x00000002) #define SND_AUDIOPROFILE_WMA9 ((__u32) 0x00000004) #define SND_AUDIOPROFILE_WMA10 ((__u32) 0x00000008) +#define SND_AUDIOPROFILE_WMA9_PRO ((__u32) 0x00000010) +#define SND_AUDIOPROFILE_WMA9_LOSSLESS ((__u32) 0x00000020) +#define SND_AUDIOPROFILE_WMA10_LOSSLESS ((__u32) 0x00000040) #define SND_AUDIOMODE_WMA_LEVEL1 ((__u32) 0x00000001) #define SND_AUDIOMODE_WMA_LEVEL2 ((__u32) 0x00000002) @@ -326,6 +331,33 @@ struct snd_dec_flac { __u16 reserved; } __attribute__((packed, aligned(4))); +struct snd_dec_wma { + __u32 encoder_option; + __u32 adv_encoder_option; + __u32 adv_encoder_option2; + __u32 reserved; +} __attribute__((packed, aligned(4))); + +struct snd_dec_alac { + __u32 frame_length; + __u8 compatible_version; + __u8 pb; + __u8 mb; + __u8 kb; + __u32 max_run; + __u32 max_frame_bytes; +} __attribute__((packed, aligned(4))); + +struct snd_dec_ape { + __u16 compatible_version; + __u16 compression_level; + __u32 format_flags; + __u32 blocks_per_frame; + __u32 final_frame_blocks; + __u32 total_frames; + __u32 seek_table_present; +} __attribute__((packed, aligned(4))); + union snd_codec_options { struct snd_enc_wma wma; struct snd_enc_vorbis vorbis; @@ -333,6 +365,9 @@ union snd_codec_options { struct snd_enc_flac flac; struct snd_enc_generic generic; struct snd_dec_flac flac_d; + struct snd_dec_wma wma_d; + struct snd_dec_alac alac_d; + struct snd_dec_ape ape_d; } __attribute__((packed, aligned(4))); /** struct snd_codec_desc - description of codec capabilities diff --git a/include/uapi/sound/sof/abi.h b/include/uapi/sound/sof/abi.h index c0ef1643c753..5995b79d6df1 100644 --- a/include/uapi/sound/sof/abi.h +++ b/include/uapi/sound/sof/abi.h @@ -26,7 +26,7 @@ /* SOF ABI version major, minor and patch numbers */ #define SOF_ABI_MAJOR 3 -#define SOF_ABI_MINOR 12 +#define SOF_ABI_MINOR 13 #define SOF_ABI_PATCH 0 /* SOF ABI version number. Format within 32bit word is MMmmmppp */ diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 915dda3f7f19..755c07d845ce 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -1813,12 +1813,14 @@ int cgroup_show_path(struct seq_file *sf, struct kernfs_node *kf_node, enum cgroup2_param { Opt_nsdelegate, Opt_memory_localevents, + Opt_memory_recursiveprot, nr__cgroup2_params }; static const struct fs_parameter_spec cgroup2_fs_parameters[] = { fsparam_flag("nsdelegate", Opt_nsdelegate), fsparam_flag("memory_localevents", Opt_memory_localevents), + fsparam_flag("memory_recursiveprot", Opt_memory_recursiveprot), {} }; @@ -1839,6 +1841,9 @@ static int cgroup2_parse_param(struct fs_context *fc, struct fs_parameter *param case Opt_memory_localevents: ctx->flags |= CGRP_ROOT_MEMORY_LOCAL_EVENTS; return 0; + case Opt_memory_recursiveprot: + ctx->flags |= CGRP_ROOT_MEMORY_RECURSIVE_PROT; + return 0; } return -EINVAL; } @@ -1855,6 +1860,11 @@ static void apply_cgroup_root_flags(unsigned int root_flags) cgrp_dfl_root.flags |= CGRP_ROOT_MEMORY_LOCAL_EVENTS; else cgrp_dfl_root.flags &= ~CGRP_ROOT_MEMORY_LOCAL_EVENTS; + + if (root_flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT) + cgrp_dfl_root.flags |= CGRP_ROOT_MEMORY_RECURSIVE_PROT; + else + cgrp_dfl_root.flags &= ~CGRP_ROOT_MEMORY_RECURSIVE_PROT; } } @@ -1864,6 +1874,8 @@ static int cgroup_show_options(struct seq_file *seq, struct kernfs_root *kf_root seq_puts(seq, ",nsdelegate"); if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS) seq_puts(seq, ",memory_localevents"); + if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT) + seq_puts(seq, ",memory_recursiveprot"); return 0; } @@ -6412,7 +6424,10 @@ static struct kobj_attribute cgroup_delegate_attr = __ATTR_RO(delegate); static ssize_t features_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - return snprintf(buf, PAGE_SIZE, "nsdelegate\nmemory_localevents\n"); + return snprintf(buf, PAGE_SIZE, + "nsdelegate\n" + "memory_localevents\n" + "memory_recursiveprot\n"); } static struct kobj_attribute cgroup_features_attr = __ATTR_RO(features); diff --git a/kernel/fork.c b/kernel/fork.c index 8a7873a8c8f0..edf168f34d77 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -283,7 +283,7 @@ static inline void free_thread_stack(struct task_struct *tsk) MEMCG_KERNEL_STACK_KB, -(int)(PAGE_SIZE / 1024)); - memcg_kmem_uncharge(vm->pages[i], 0); + memcg_kmem_uncharge_page(vm->pages[i], 0); } for (i = 0; i < NR_CACHED_STACKS; i++) { @@ -415,12 +415,13 @@ static int memcg_charge_kernel_stack(struct task_struct *tsk) for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { /* - * If memcg_kmem_charge() fails, page->mem_cgroup - * pointer is NULL, and both memcg_kmem_uncharge() + * If memcg_kmem_charge_page() fails, page->mem_cgroup + * pointer is NULL, and both memcg_kmem_uncharge_page() * and mod_memcg_page_state() in free_thread_stack() * will ignore this page. So it's safe. */ - ret = memcg_kmem_charge(vm->pages[i], GFP_KERNEL, 0); + ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL, + 0); if (ret) return ret; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index b815578187de..507902cc419d 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -213,6 +213,11 @@ static int proc_do_cad_pid(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); static int proc_taint(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +#ifdef CONFIG_COMPACTION +static int proc_dointvec_minmax_warn_RT_change(struct ctl_table *table, + int write, void __user *buffer, + size_t *lenp, loff_t *ppos); +#endif #endif #ifdef CONFIG_PRINTK @@ -1468,7 +1473,7 @@ static struct ctl_table vm_table[] = { .data = &sysctl_compact_unevictable_allowed, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_dointvec_minmax_warn_RT_change, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, @@ -2564,6 +2569,28 @@ int proc_dointvec(struct ctl_table *table, int write, return do_proc_dointvec(table, write, buffer, lenp, ppos, NULL, NULL); } +#ifdef CONFIG_COMPACTION +static int proc_dointvec_minmax_warn_RT_change(struct ctl_table *table, + int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret, old; + + if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write) + return proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + old = *(int *)table->data; + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + return ret; + if (old != *(int *)table->data) + pr_warn_once("sysctl attribute %s changed by %s[%d]\n", + table->procname, current->comm, + task_pid_nr(current)); + return ret; +} +#endif + /** * proc_douintvec - read a vector of unsigned integers * @table: the sysctl table diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 4f6c6ebbbbde..8d092609928e 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -50,9 +50,10 @@ static unsigned long __percpu *percpu_count_ptr(struct percpu_ref *ref) * @flags: PERCPU_REF_INIT_* flags * @gfp: allocation mask to use * - * Initializes @ref. If @flags is zero, @ref starts in percpu mode with a - * refcount of 1; analagous to atomic_long_set(ref, 1). See the - * definitions of PERCPU_REF_INIT_* flags for flag behaviors. + * Initializes @ref. @ref starts out in percpu mode with a refcount of 1 unless + * @flags contains PERCPU_REF_INIT_ATOMIC or PERCPU_REF_INIT_DEAD. These flags + * change the start state to atomic with the latter setting the initial refcount + * to 0. See the definitions of PERCPU_REF_INIT_* flags for flag behaviors. * * Note that @release must not sleep - it may potentially be called from RCU * callback context by percpu_ref_kill(). diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 3872d250ed2c..e3087d90e00d 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -285,6 +285,24 @@ static noinline void __init kmalloc_oob_in_memset(void) kfree(ptr); } +static noinline void __init kmalloc_memmove_invalid_size(void) +{ + char *ptr; + size_t size = 64; + volatile size_t invalid_size = -2; + + pr_info("invalid size in memmove\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + memset((char *)ptr, 0, 64); + memmove((char *)ptr, (char *)ptr + 4, invalid_size); + kfree(ptr); +} + static noinline void __init kmalloc_uaf(void) { char *ptr; @@ -799,6 +817,7 @@ static int __init kmalloc_tests_init(void) kmalloc_oob_memset_4(); kmalloc_oob_memset_8(); kmalloc_oob_memset_16(); + kmalloc_memmove_invalid_size(); kmalloc_uaf(); kmalloc_uaf_memset(); kmalloc_uaf2(); diff --git a/mm/Makefile b/mm/Makefile index 272e66039e70..dbc8346d16ca 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -6,6 +6,7 @@ KASAN_SANITIZE_slab_common.o := n KASAN_SANITIZE_slab.o := n KASAN_SANITIZE_slub.o := n +KCSAN_SANITIZE_kmemleak.o := n # These files are disabled because they produce non-interesting and/or # flaky coverage that is not a function of syscall inputs. E.g. slab is out of diff --git a/mm/compaction.c b/mm/compaction.c index 672d3c78c6ab..df3da2f76fdc 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -894,12 +894,13 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, /* * Regardless of being on LRU, compound pages such as THP and - * hugetlbfs are not to be compacted. We can potentially save - * a lot of iterations if we skip them at once. The check is - * racy, but we can consider only valid values and the only - * danger is skipping too much. + * hugetlbfs are not to be compacted unless we are attempting + * an allocation much larger than the huge page size (eg CMA). + * We can potentially save a lot of iterations if we skip them + * at once. The check is racy, but we can consider only valid + * values and the only danger is skipping too much. */ - if (PageCompound(page)) { + if (PageCompound(page) && !cc->alloc_contig) { const unsigned int order = compound_order(page); if (likely(order < MAX_ORDER)) @@ -969,7 +970,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, * and it's on LRU. It can only be a THP so the order * is safe to read and it's 0 for tail pages. */ - if (unlikely(PageCompound(page))) { + if (unlikely(PageCompound(page) && !cc->alloc_contig)) { low_pfn += compound_nr(page) - 1; goto isolate_fail; } @@ -981,12 +982,15 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, if (__isolate_lru_page(page, isolate_mode) != 0) goto isolate_fail; - VM_BUG_ON_PAGE(PageCompound(page), page); + /* The whole page is taken off the LRU; skip the tail pages. */ + if (PageCompound(page)) + low_pfn += compound_nr(page) - 1; /* Successfully isolated */ del_page_from_lru_list(page, lruvec, page_lru(page)); - inc_node_page_state(page, - NR_ISOLATED_ANON + page_is_file_cache(page)); + mod_node_page_state(page_pgdat(page), + NR_ISOLATED_ANON + page_is_file_cache(page), + hpage_nr_pages(page)); isolate_success: list_add(&page->lru, &cc->migratepages); @@ -1590,7 +1594,11 @@ typedef enum { * Allow userspace to control policy on scanning the unevictable LRU for * compactable pages. */ +#ifdef CONFIG_PREEMPT_RT +int sysctl_compact_unevictable_allowed __read_mostly = 0; +#else int sysctl_compact_unevictable_allowed __read_mostly = 1; +#endif static inline void update_fast_start_pfn(struct compact_control *cc, unsigned long pfn) @@ -2174,7 +2182,6 @@ compact_zone(struct compact_control *cc, struct capture_control *capc) ret = COMPACT_CONTENDED; putback_movable_pages(&cc->migratepages); cc->nr_migratepages = 0; - last_migrated_pfn = 0; goto out; case ISOLATE_NONE: if (update_cached) { @@ -2310,8 +2317,7 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, .page = NULL, }; - if (capture) - current->capture_control = &capc; + current->capture_control = &capc; ret = compact_zone(&cc, &capc); @@ -2333,6 +2339,7 @@ int sysctl_extfrag_threshold = 500; * @alloc_flags: The allocation flags of the current allocation * @ac: The context of current allocation * @prio: Determines how hard direct compaction should try to succeed + * @capture: Pointer to free page created by compaction will be stored here * * This is the main entry point for direct page compaction. */ diff --git a/mm/debug.c b/mm/debug.c index ecccd9f17801..2189357f0987 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -44,8 +44,10 @@ const struct trace_print_flags vmaflag_names[] = { void __dump_page(struct page *page, const char *reason) { + struct page *head = compound_head(page); struct address_space *mapping; bool page_poisoned = PagePoisoned(page); + bool compound = PageCompound(page); /* * Accessing the pageblock without the zone lock. It could change to * "isolate" again in the meantime, but since we are just dumping the @@ -66,25 +68,43 @@ void __dump_page(struct page *page, const char *reason) goto hex_only; } - mapping = page_mapping(page); + if (page < head || (page >= head + MAX_ORDER_NR_PAGES)) { + /* Corrupt page, cannot call page_mapping */ + mapping = page->mapping; + head = page; + compound = false; + } else { + mapping = page_mapping(page); + } /* * Avoid VM_BUG_ON() in page_mapcount(). * page->_mapcount space in struct page is used by sl[aou]b pages to * encode own info. */ - mapcount = PageSlab(page) ? 0 : page_mapcount(page); + mapcount = PageSlab(head) ? 0 : page_mapcount(page); - if (PageCompound(page)) - pr_warn("page:%px refcount:%d mapcount:%d mapping:%px " - "index:%#lx compound_mapcount: %d\n", - page, page_ref_count(page), mapcount, - page->mapping, page_to_pgoff(page), - compound_mapcount(page)); + if (compound) + if (hpage_pincount_available(page)) { + pr_warn("page:%px refcount:%d mapcount:%d mapping:%p " + "index:%#lx head:%px order:%u " + "compound_mapcount:%d compound_pincount:%d\n", + page, page_ref_count(head), mapcount, + mapping, page_to_pgoff(page), head, + compound_order(head), compound_mapcount(page), + compound_pincount(page)); + } else { + pr_warn("page:%px refcount:%d mapcount:%d mapping:%p " + "index:%#lx head:%px order:%u " + "compound_mapcount:%d\n", + page, page_ref_count(head), mapcount, + mapping, page_to_pgoff(page), head, + compound_order(head), compound_mapcount(page)); + } else - pr_warn("page:%px refcount:%d mapcount:%d mapping:%px index:%#lx\n", + pr_warn("page:%px refcount:%d mapcount:%d mapping:%p index:%#lx\n", page, page_ref_count(page), mapcount, - page->mapping, page_to_pgoff(page)); + mapping, page_to_pgoff(page)); if (PageKsm(page)) type = "ksm "; else if (PageAnon(page)) @@ -106,6 +126,10 @@ void __dump_page(struct page *page, const char *reason) print_hex_dump(KERN_WARNING, "raw: ", DUMP_PREFIX_NONE, 32, sizeof(unsigned long), page, sizeof(struct page), false); + if (head != page) + print_hex_dump(KERN_WARNING, "head: ", DUMP_PREFIX_NONE, 32, + sizeof(unsigned long), head, + sizeof(struct page), false); if (reason) pr_warn("page dumped because: %s\n", reason); diff --git a/mm/filemap.c b/mm/filemap.c index 1784478270e1..0fbdc8e30dd2 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1386,7 +1386,7 @@ EXPORT_SYMBOL_GPL(__lock_page_killable); int __lock_page_or_retry(struct page *page, struct mm_struct *mm, unsigned int flags) { - if (flags & FAULT_FLAG_ALLOW_RETRY) { + if (fault_flag_allow_retry_first(flags)) { /* * CAUTION! In this case, mmap_sem is not released * even though return 0. @@ -1536,7 +1536,6 @@ struct page *find_get_entry(struct address_space *mapping, pgoff_t offset) return page; } -EXPORT_SYMBOL(find_get_entry); /** * find_lock_entry - locate, pin and lock a page cache entry @@ -1575,42 +1574,39 @@ struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset) EXPORT_SYMBOL(find_lock_entry); /** - * pagecache_get_page - find and get a page reference - * @mapping: the address_space to search - * @offset: the page index - * @fgp_flags: PCG flags - * @gfp_mask: gfp mask to use for the page cache data page allocation + * pagecache_get_page - Find and get a reference to a page. + * @mapping: The address_space to search. + * @index: The page index. + * @fgp_flags: %FGP flags modify how the page is returned. + * @gfp_mask: Memory allocation flags to use if %FGP_CREAT is specified. * - * Looks up the page cache slot at @mapping & @offset. + * Looks up the page cache entry at @mapping & @index. * - * PCG flags modify how the page is returned. + * @fgp_flags can be zero or more of these flags: * - * @fgp_flags can be: + * * %FGP_ACCESSED - The page will be marked accessed. + * * %FGP_LOCK - The page is returned locked. + * * %FGP_CREAT - If no page is present then a new page is allocated using + * @gfp_mask and added to the page cache and the VM's LRU list. + * The page is returned locked and with an increased refcount. + * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the + * page is already in cache. If the page was allocated, unlock it before + * returning so the caller can do the same dance. * - * - FGP_ACCESSED: the page will be marked accessed - * - FGP_LOCK: Page is return locked - * - FGP_CREAT: If page is not present then a new page is allocated using - * @gfp_mask and added to the page cache and the VM's LRU - * list. The page is returned locked and with an increased - * refcount. - * - FGP_FOR_MMAP: Similar to FGP_CREAT, only we want to allow the caller to do - * its own locking dance if the page is already in cache, or unlock the page - * before returning if we had to add the page to pagecache. - * - * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even - * if the GFP flags specified for FGP_CREAT are atomic. + * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even + * if the %GFP flags specified for %FGP_CREAT are atomic. * * If there is a page cache page, it is returned with an increased refcount. * - * Return: the found page or %NULL otherwise. + * Return: The found page or %NULL otherwise. */ -struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset, - int fgp_flags, gfp_t gfp_mask) +struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index, + int fgp_flags, gfp_t gfp_mask) { struct page *page; repeat: - page = find_get_entry(mapping, offset); + page = find_get_entry(mapping, index); if (xa_is_value(page)) page = NULL; if (!page) @@ -1632,7 +1628,7 @@ struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset, put_page(page); goto repeat; } - VM_BUG_ON_PAGE(page->index != offset, page); + VM_BUG_ON_PAGE(page->index != index, page); } if (fgp_flags & FGP_ACCESSED) @@ -1657,7 +1653,7 @@ struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset, if (fgp_flags & FGP_ACCESSED) __SetPageReferenced(page); - err = add_to_page_cache_lru(page, mapping, offset, gfp_mask); + err = add_to_page_cache_lru(page, mapping, index, gfp_mask); if (unlikely(err)) { put_page(page); page = NULL; @@ -1962,8 +1958,7 @@ EXPORT_SYMBOL(find_get_pages_range_tag); * * It is going insane. Fix it by quickly scaling down the readahead size. */ -static void shrink_readahead_size_eio(struct file *filp, - struct file_ra_state *ra) +static void shrink_readahead_size_eio(struct file_ra_state *ra) { ra->ra_pages /= 4; } @@ -2188,7 +2183,7 @@ static ssize_t generic_file_buffered_read(struct kiocb *iocb, goto find_page; } unlock_page(page); - shrink_readahead_size_eio(filp, ra); + shrink_readahead_size_eio(ra); error = -EIO; goto readpage_error; } @@ -2416,7 +2411,7 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf, pgoff_t offset = vmf->pgoff; /* If we don't want any read-ahead, don't bother */ - if (vmf->vma->vm_flags & VM_RAND_READ) + if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) return fpin; if (ra->mmap_miss > 0) ra->mmap_miss--; @@ -2491,7 +2486,7 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) if (!page) { if (fpin) goto out_retry; - return vmf_error(-ENOMEM); + return VM_FAULT_OOM; } } @@ -2560,7 +2555,7 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) goto retry_find; /* Things didn't work out. Return zero to tell the mm layer so. */ - shrink_readahead_size_eio(file, ra); + shrink_readahead_size_eio(ra); return VM_FAULT_SIGBUS; out_retry: @@ -2823,6 +2818,14 @@ static struct page *do_read_cache_page(struct address_space *mapping, unlock_page(page); goto out; } + + /* + * A previous I/O error may have been due to temporary + * failures. + * Clear page error before actual read, PG_error will be + * set again if read page fails. + */ + ClearPageError(page); goto filler; out: diff --git a/mm/gup.c b/mm/gup.c index 1b521e0ac1de..da3e03185144 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -29,6 +29,22 @@ struct follow_page_context { unsigned int page_mask; }; +static void hpage_pincount_add(struct page *page, int refs) +{ + VM_BUG_ON_PAGE(!hpage_pincount_available(page), page); + VM_BUG_ON_PAGE(page != compound_head(page), page); + + atomic_add(refs, compound_pincount_ptr(page)); +} + +static void hpage_pincount_sub(struct page *page, int refs) +{ + VM_BUG_ON_PAGE(!hpage_pincount_available(page), page); + VM_BUG_ON_PAGE(page != compound_head(page), page); + + atomic_sub(refs, compound_pincount_ptr(page)); +} + /* * Return the compound head page with ref appropriately incremented, * or NULL if that failed. @@ -44,6 +60,195 @@ static inline struct page *try_get_compound_head(struct page *page, int refs) return head; } +/* + * try_grab_compound_head() - attempt to elevate a page's refcount, by a + * flags-dependent amount. + * + * "grab" names in this file mean, "look at flags to decide whether to use + * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount. + * + * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the + * same time. (That's true throughout the get_user_pages*() and + * pin_user_pages*() APIs.) Cases: + * + * FOLL_GET: page's refcount will be incremented by 1. + * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS. + * + * Return: head page (with refcount appropriately incremented) for success, or + * NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's + * considered failure, and furthermore, a likely bug in the caller, so a warning + * is also emitted. + */ +static __maybe_unused struct page *try_grab_compound_head(struct page *page, + int refs, + unsigned int flags) +{ + if (flags & FOLL_GET) + return try_get_compound_head(page, refs); + else if (flags & FOLL_PIN) { + int orig_refs = refs; + + /* + * Can't do FOLL_LONGTERM + FOLL_PIN with CMA in the gup fast + * path, so fail and let the caller fall back to the slow path. + */ + if (unlikely(flags & FOLL_LONGTERM) && + is_migrate_cma_page(page)) + return NULL; + + /* + * When pinning a compound page of order > 1 (which is what + * hpage_pincount_available() checks for), use an exact count to + * track it, via hpage_pincount_add/_sub(). + * + * However, be sure to *also* increment the normal page refcount + * field at least once, so that the page really is pinned. + */ + if (!hpage_pincount_available(page)) + refs *= GUP_PIN_COUNTING_BIAS; + + page = try_get_compound_head(page, refs); + if (!page) + return NULL; + + if (hpage_pincount_available(page)) + hpage_pincount_add(page, refs); + + mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, + orig_refs); + + return page; + } + + WARN_ON_ONCE(1); + return NULL; +} + +/** + * try_grab_page() - elevate a page's refcount by a flag-dependent amount + * + * This might not do anything at all, depending on the flags argument. + * + * "grab" names in this file mean, "look at flags to decide whether to use + * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount. + * + * @page: pointer to page to be grabbed + * @flags: gup flags: these are the FOLL_* flag values. + * + * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same + * time. Cases: + * + * FOLL_GET: page's refcount will be incremented by 1. + * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS. + * + * Return: true for success, or if no action was required (if neither FOLL_PIN + * nor FOLL_GET was set, nothing is done). False for failure: FOLL_GET or + * FOLL_PIN was set, but the page could not be grabbed. + */ +bool __must_check try_grab_page(struct page *page, unsigned int flags) +{ + WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == (FOLL_GET | FOLL_PIN)); + + if (flags & FOLL_GET) + return try_get_page(page); + else if (flags & FOLL_PIN) { + int refs = 1; + + page = compound_head(page); + + if (WARN_ON_ONCE(page_ref_count(page) <= 0)) + return false; + + if (hpage_pincount_available(page)) + hpage_pincount_add(page, 1); + else + refs = GUP_PIN_COUNTING_BIAS; + + /* + * Similar to try_grab_compound_head(): even if using the + * hpage_pincount_add/_sub() routines, be sure to + * *also* increment the normal page refcount field at least + * once, so that the page really is pinned. + */ + page_ref_add(page, refs); + + mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1); + } + + return true; +} + +#ifdef CONFIG_DEV_PAGEMAP_OPS +static bool __unpin_devmap_managed_user_page(struct page *page) +{ + int count, refs = 1; + + if (!page_is_devmap_managed(page)) + return false; + + if (hpage_pincount_available(page)) + hpage_pincount_sub(page, 1); + else + refs = GUP_PIN_COUNTING_BIAS; + + count = page_ref_sub_return(page, refs); + + mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED, 1); + /* + * devmap page refcounts are 1-based, rather than 0-based: if + * refcount is 1, then the page is free and the refcount is + * stable because nobody holds a reference on the page. + */ + if (count == 1) + free_devmap_managed_page(page); + else if (!count) + __put_page(page); + + return true; +} +#else +static bool __unpin_devmap_managed_user_page(struct page *page) +{ + return false; +} +#endif /* CONFIG_DEV_PAGEMAP_OPS */ + +/** + * unpin_user_page() - release a dma-pinned page + * @page: pointer to page to be released + * + * Pages that were pinned via pin_user_pages*() must be released via either + * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so + * that such pages can be separately tracked and uniquely handled. In + * particular, interactions with RDMA and filesystems need special handling. + */ +void unpin_user_page(struct page *page) +{ + int refs = 1; + + page = compound_head(page); + + /* + * For devmap managed pages we need to catch refcount transition from + * GUP_PIN_COUNTING_BIAS to 1, when refcount reach one it means the + * page is free and we need to inform the device driver through + * callback. See include/linux/memremap.h and HMM for details. + */ + if (__unpin_devmap_managed_user_page(page)) + return; + + if (hpage_pincount_available(page)) + hpage_pincount_sub(page, 1); + else + refs = GUP_PIN_COUNTING_BIAS; + + if (page_ref_sub_and_test(page, refs)) + __put_page(page); + + mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED, 1); +} +EXPORT_SYMBOL(unpin_user_page); + /** * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages * @pages: array of pages to be maybe marked dirty, and definitely released. @@ -193,6 +398,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, struct page *page; spinlock_t *ptl; pte_t *ptep, pte; + int ret; /* FOLL_GET and FOLL_PIN are mutually exclusive. */ if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == @@ -230,10 +436,11 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, } page = vm_normal_page(vma, address, pte); - if (!page && pte_devmap(pte) && (flags & FOLL_GET)) { + if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) { /* - * Only return device mapping pages in the FOLL_GET case since - * they are only valid while holding the pgmap reference. + * Only return device mapping pages in the FOLL_GET or FOLL_PIN + * case since they are only valid while holding the pgmap + * reference. */ *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap); if (*pgmap) @@ -250,8 +457,6 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, if (is_zero_pfn(pte_pfn(pte))) { page = pte_page(pte); } else { - int ret; - ret = follow_pfn_pte(vma, address, ptep, flags); page = ERR_PTR(ret); goto out; @@ -259,7 +464,6 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, } if (flags & FOLL_SPLIT && PageTransCompound(page)) { - int ret; get_page(page); pte_unmap_unlock(ptep, ptl); lock_page(page); @@ -271,9 +475,21 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, goto retry; } - if (flags & FOLL_GET) { - if (unlikely(!try_get_page(page))) { - page = ERR_PTR(-ENOMEM); + /* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */ + if (unlikely(!try_grab_page(page, flags))) { + page = ERR_PTR(-ENOMEM); + goto out; + } + /* + * We need to make the page accessible if and only if we are going + * to access its content (the FOLL_PIN case). Please see + * Documentation/core-api/pin_user_pages.rst for details. + */ + if (flags & FOLL_PIN) { + ret = arch_make_page_accessible(page); + if (ret) { + unpin_user_page(page); + page = ERR_PTR(ret); goto out; } } @@ -537,7 +753,7 @@ static struct page *follow_page_mask(struct vm_area_struct *vma, /* make this handle hugepd */ page = follow_huge_addr(mm, address, flags & FOLL_WRITE); if (!IS_ERR(page)) { - BUG_ON(flags & FOLL_GET); + WARN_ON_ONCE(flags & (FOLL_GET | FOLL_PIN)); return page; } @@ -630,12 +846,12 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address, } /* - * mmap_sem must be held on entry. If @nonblocking != NULL and - * *@flags does not include FOLL_NOWAIT, the mmap_sem may be released. - * If it is, *@nonblocking will be set to 0 and -EBUSY returned. + * mmap_sem must be held on entry. If @locked != NULL and *@flags + * does not include FOLL_NOWAIT, the mmap_sem may be released. If it + * is, *@locked will be set to 0 and -EBUSY returned. */ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, - unsigned long address, unsigned int *flags, int *nonblocking) + unsigned long address, unsigned int *flags, int *locked) { unsigned int fault_flags = 0; vm_fault_t ret; @@ -647,12 +863,15 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, fault_flags |= FAULT_FLAG_WRITE; if (*flags & FOLL_REMOTE) fault_flags |= FAULT_FLAG_REMOTE; - if (nonblocking) - fault_flags |= FAULT_FLAG_ALLOW_RETRY; + if (locked) + fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; if (*flags & FOLL_NOWAIT) fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT; if (*flags & FOLL_TRIED) { - VM_WARN_ON_ONCE(fault_flags & FAULT_FLAG_ALLOW_RETRY); + /* + * Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED + * can co-exist + */ fault_flags |= FAULT_FLAG_TRIED; } @@ -673,8 +892,8 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, } if (ret & VM_FAULT_RETRY) { - if (nonblocking && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT)) - *nonblocking = 0; + if (locked && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT)) + *locked = 0; return -EBUSY; } @@ -751,7 +970,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) * only intends to ensure the pages are faulted in. * @vmas: array of pointers to vmas corresponding to each page. * Or NULL if the caller does not require them. - * @nonblocking: whether waiting for disk IO or mmap_sem contention + * @locked: whether we're still with the mmap_sem held * * Returns either number of pages pinned (which may be less than the * number requested), or an error. Details about the return value: @@ -786,13 +1005,11 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) * appropriate) must be called after the page is finished with, and * before put_page is called. * - * If @nonblocking != NULL, __get_user_pages will not wait for disk IO - * or mmap_sem contention, and if waiting is needed to pin all pages, - * *@nonblocking will be set to 0. Further, if @gup_flags does not - * include FOLL_NOWAIT, the mmap_sem will be released via up_read() in - * this case. + * If @locked != NULL, *@locked will be set to 0 when mmap_sem is + * released by an up_read(). That can happen if @gup_flags does not + * have FOLL_NOWAIT. * - * A caller using such a combination of @nonblocking and @gup_flags + * A caller using such a combination of @locked and @gup_flags * must therefore hold the mmap_sem for reading only, and recognize * when it's been released. Otherwise, it must be held for either * reading or writing and will not be released. @@ -804,7 +1021,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, - struct vm_area_struct **vmas, int *nonblocking) + struct vm_area_struct **vmas, int *locked) { long ret = 0, i = 0; struct vm_area_struct *vma = NULL; @@ -850,7 +1067,17 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, if (is_vm_hugetlb_page(vma)) { i = follow_hugetlb_page(mm, vma, pages, vmas, &start, &nr_pages, i, - gup_flags, nonblocking); + gup_flags, locked); + if (locked && *locked == 0) { + /* + * We've got a VM_FAULT_RETRY + * and we've lost mmap_sem. + * We must stop here. + */ + BUG_ON(gup_flags & FOLL_NOWAIT); + BUG_ON(ret != 0); + goto out; + } continue; } } @@ -868,7 +1095,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, page = follow_page_mask(vma, start, foll_flags, &ctx); if (!page) { ret = faultin_page(tsk, vma, start, &foll_flags, - nonblocking); + locked); switch (ret) { case 0: goto retry; @@ -980,7 +1207,7 @@ int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, address = untagged_addr(address); if (unlocked) - fault_flags |= FAULT_FLAG_ALLOW_RETRY; + fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; retry: vma = find_extend_vma(mm, address); @@ -1004,7 +1231,6 @@ int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, down_read(&mm->mmap_sem); if (!(fault_flags & FAULT_FLAG_TRIED)) { *unlocked = true; - fault_flags &= ~FAULT_FLAG_ALLOW_RETRY; fault_flags |= FAULT_FLAG_TRIED; goto retry; } @@ -1088,17 +1314,36 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk, if (likely(pages)) pages += ret; start += ret << PAGE_SHIFT; + lock_dropped = true; +retry: /* * Repeat on the address that fired VM_FAULT_RETRY - * without FAULT_FLAG_ALLOW_RETRY but with - * FAULT_FLAG_TRIED. + * with both FAULT_FLAG_ALLOW_RETRY and + * FAULT_FLAG_TRIED. Note that GUP can be interrupted + * by fatal signals, so we need to check it before we + * start trying again otherwise it can loop forever. */ + + if (fatal_signal_pending(current)) + break; + *locked = 1; - lock_dropped = true; - down_read(&mm->mmap_sem); + ret = down_read_killable(&mm->mmap_sem); + if (ret) { + BUG_ON(ret > 0); + if (!pages_done) + pages_done = ret; + break; + } + ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED, - pages, NULL, NULL); + pages, NULL, locked); + if (!*locked) { + /* Continue to retry until we succeeded */ + BUG_ON(ret != 0); + goto retry; + } if (ret != 1) { BUG_ON(ret > 1); if (!pages_done) @@ -1129,7 +1374,7 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk, * @vma: target vma * @start: start address * @end: end address - * @nonblocking: + * @locked: whether the mmap_sem is still held * * This takes care of mlocking the pages too if VM_LOCKED is set. * @@ -1137,14 +1382,14 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk, * * vma->vm_mm->mmap_sem must be held. * - * If @nonblocking is NULL, it may be held for read or write and will + * If @locked is NULL, it may be held for read or write and will * be unperturbed. * - * If @nonblocking is non-NULL, it must held for read only and may be - * released. If it's released, *@nonblocking will be set to 0. + * If @locked is non-NULL, it must held for read only and may be + * released. If it's released, *@locked will be set to 0. */ long populate_vma_page_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, int *nonblocking) + unsigned long start, unsigned long end, int *locked) { struct mm_struct *mm = vma->vm_mm; unsigned long nr_pages = (end - start) / PAGE_SIZE; @@ -1179,7 +1424,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, * not result in a stack expansion that recurses back here. */ return __get_user_pages(current, mm, start, nr_pages, gup_flags, - NULL, NULL, nonblocking); + NULL, NULL, locked); } /* @@ -1557,6 +1802,37 @@ static __always_inline long __gup_longterm_locked(struct task_struct *tsk, } #endif /* CONFIG_FS_DAX || CONFIG_CMA */ +#ifdef CONFIG_MMU +static long __get_user_pages_remote(struct task_struct *tsk, + struct mm_struct *mm, + unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages, + struct vm_area_struct **vmas, int *locked) +{ + /* + * Parts of FOLL_LONGTERM behavior are incompatible with + * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on + * vmas. However, this only comes up if locked is set, and there are + * callers that do request FOLL_LONGTERM, but do not set locked. So, + * allow what we can. + */ + if (gup_flags & FOLL_LONGTERM) { + if (WARN_ON_ONCE(locked)) + return -EINVAL; + /* + * This will check the vmas (even if our vmas arg is NULL) + * and return -ENOTSUPP if DAX isn't allowed in this case: + */ + return __gup_longterm_locked(tsk, mm, start, nr_pages, pages, + vmas, gup_flags | FOLL_TOUCH | + FOLL_REMOTE); + } + + return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas, + locked, + gup_flags | FOLL_TOUCH | FOLL_REMOTE); +} + /* * get_user_pages_remote() - pin user pages in memory * @tsk: the task_struct to use for page fault accounting, or @@ -1619,7 +1895,6 @@ static __always_inline long __gup_longterm_locked(struct task_struct *tsk, * should use get_user_pages because it cannot pass * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault. */ -#ifdef CONFIG_MMU long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, @@ -1632,28 +1907,8 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) return -EINVAL; - /* - * Parts of FOLL_LONGTERM behavior are incompatible with - * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on - * vmas. However, this only comes up if locked is set, and there are - * callers that do request FOLL_LONGTERM, but do not set locked. So, - * allow what we can. - */ - if (gup_flags & FOLL_LONGTERM) { - if (WARN_ON_ONCE(locked)) - return -EINVAL; - /* - * This will check the vmas (even if our vmas arg is NULL) - * and return -ENOTSUPP if DAX isn't allowed in this case: - */ - return __gup_longterm_locked(tsk, mm, start, nr_pages, pages, - vmas, gup_flags | FOLL_TOUCH | - FOLL_REMOTE); - } - - return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas, - locked, - gup_flags | FOLL_TOUCH | FOLL_REMOTE); + return __get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags, + pages, vmas, locked); } EXPORT_SYMBOL(get_user_pages_remote); @@ -1665,6 +1920,15 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, { return 0; } + +static long __get_user_pages_remote(struct task_struct *tsk, + struct mm_struct *mm, + unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages, + struct vm_area_struct **vmas, int *locked) +{ + return 0; +} #endif /* !CONFIG_MMU */ /* @@ -1804,7 +2068,31 @@ EXPORT_SYMBOL(get_user_pages_unlocked); * This code is based heavily on the PowerPC implementation by Nick Piggin. */ #ifdef CONFIG_HAVE_FAST_GUP + +static void put_compound_head(struct page *page, int refs, unsigned int flags) +{ + if (flags & FOLL_PIN) { + mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED, + refs); + + if (hpage_pincount_available(page)) + hpage_pincount_sub(page, refs); + else + refs *= GUP_PIN_COUNTING_BIAS; + } + + VM_BUG_ON_PAGE(page_ref_count(page) < refs, page); + /* + * Calling put_page() for each ref is unnecessarily slow. Only the last + * ref needs a put_page(). + */ + if (refs > 1) + page_ref_sub(page, refs - 1); + put_page(page); +} + #ifdef CONFIG_GUP_GET_PTE_LOW_HIGH + /* * WARNING: only to be used in the get_user_pages_fast() implementation. * @@ -1860,13 +2148,17 @@ static inline pte_t gup_get_pte(pte_t *ptep) #endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */ static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start, + unsigned int flags, struct page **pages) { while ((*nr) - nr_start) { struct page *page = pages[--(*nr)]; ClearPageReferenced(page); - put_page(page); + if (flags & FOLL_PIN) + unpin_user_page(page); + else + put_page(page); } } @@ -1899,7 +2191,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, pgmap = get_dev_pagemap(pte_pfn(pte), pgmap); if (unlikely(!pgmap)) { - undo_dev_pagemap(nr, nr_start, pages); + undo_dev_pagemap(nr, nr_start, flags, pages); goto pte_unmap; } } else if (pte_special(pte)) @@ -1908,17 +2200,30 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, VM_BUG_ON(!pfn_valid(pte_pfn(pte))); page = pte_page(pte); - head = try_get_compound_head(page, 1); + head = try_grab_compound_head(page, 1, flags); if (!head) goto pte_unmap; if (unlikely(pte_val(pte) != pte_val(*ptep))) { - put_page(head); + put_compound_head(head, 1, flags); goto pte_unmap; } VM_BUG_ON_PAGE(compound_head(page) != head, page); + /* + * We need to make the page accessible if and only if we are + * going to access its content (the FOLL_PIN case). Please + * see Documentation/core-api/pin_user_pages.rst for + * details. + */ + if (flags & FOLL_PIN) { + ret = arch_make_page_accessible(page); + if (ret) { + unpin_user_page(page); + goto pte_unmap; + } + } SetPageReferenced(page); pages[*nr] = page; (*nr)++; @@ -1953,7 +2258,8 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE) static int __gup_device_huge(unsigned long pfn, unsigned long addr, - unsigned long end, struct page **pages, int *nr) + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { int nr_start = *nr; struct dev_pagemap *pgmap = NULL; @@ -1963,12 +2269,15 @@ static int __gup_device_huge(unsigned long pfn, unsigned long addr, pgmap = get_dev_pagemap(pfn, pgmap); if (unlikely(!pgmap)) { - undo_dev_pagemap(nr, nr_start, pages); + undo_dev_pagemap(nr, nr_start, flags, pages); return 0; } SetPageReferenced(page); pages[*nr] = page; - get_page(page); + if (unlikely(!try_grab_page(page, flags))) { + undo_dev_pagemap(nr, nr_start, flags, pages); + return 0; + } (*nr)++; pfn++; } while (addr += PAGE_SIZE, addr != end); @@ -1979,48 +2288,52 @@ static int __gup_device_huge(unsigned long pfn, unsigned long addr, } static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, - unsigned long end, struct page **pages, int *nr) + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { unsigned long fault_pfn; int nr_start = *nr; fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); - if (!__gup_device_huge(fault_pfn, addr, end, pages, nr)) + if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) return 0; if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { - undo_dev_pagemap(nr, nr_start, pages); + undo_dev_pagemap(nr, nr_start, flags, pages); return 0; } return 1; } static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, - unsigned long end, struct page **pages, int *nr) + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { unsigned long fault_pfn; int nr_start = *nr; fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); - if (!__gup_device_huge(fault_pfn, addr, end, pages, nr)) + if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) return 0; if (unlikely(pud_val(orig) != pud_val(*pudp))) { - undo_dev_pagemap(nr, nr_start, pages); + undo_dev_pagemap(nr, nr_start, flags, pages); return 0; } return 1; } #else static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, - unsigned long end, struct page **pages, int *nr) + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { BUILD_BUG(); return 0; } static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr, - unsigned long end, struct page **pages, int *nr) + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { BUILD_BUG(); return 0; @@ -2038,18 +2351,6 @@ static int record_subpages(struct page *page, unsigned long addr, return nr; } -static void put_compound_head(struct page *page, int refs) -{ - VM_BUG_ON_PAGE(page_ref_count(page) < refs, page); - /* - * Calling put_page() for each ref is unnecessarily slow. Only the last - * ref needs a put_page(). - */ - if (refs > 1) - page_ref_sub(page, refs - 1); - put_page(page); -} - #ifdef CONFIG_ARCH_HAS_HUGEPD static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end, unsigned long sz) @@ -2083,12 +2384,12 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, page = head + ((addr & (sz-1)) >> PAGE_SHIFT); refs = record_subpages(page, addr, end, pages + *nr); - head = try_get_compound_head(head, refs); + head = try_grab_compound_head(head, refs, flags); if (!head) return 0; if (unlikely(pte_val(pte) != pte_val(*ptep))) { - put_compound_head(head, refs); + put_compound_head(head, refs, flags); return 0; } @@ -2136,18 +2437,19 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, if (pmd_devmap(orig)) { if (unlikely(flags & FOLL_LONGTERM)) return 0; - return __gup_device_huge_pmd(orig, pmdp, addr, end, pages, nr); + return __gup_device_huge_pmd(orig, pmdp, addr, end, flags, + pages, nr); } page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); refs = record_subpages(page, addr, end, pages + *nr); - head = try_get_compound_head(pmd_page(orig), refs); + head = try_grab_compound_head(pmd_page(orig), refs, flags); if (!head) return 0; if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { - put_compound_head(head, refs); + put_compound_head(head, refs, flags); return 0; } @@ -2157,7 +2459,8 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, } static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, int *nr) + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { struct page *head, *page; int refs; @@ -2168,18 +2471,19 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, if (pud_devmap(orig)) { if (unlikely(flags & FOLL_LONGTERM)) return 0; - return __gup_device_huge_pud(orig, pudp, addr, end, pages, nr); + return __gup_device_huge_pud(orig, pudp, addr, end, flags, + pages, nr); } page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); refs = record_subpages(page, addr, end, pages + *nr); - head = try_get_compound_head(pud_page(orig), refs); + head = try_grab_compound_head(pud_page(orig), refs, flags); if (!head) return 0; if (unlikely(pud_val(orig) != pud_val(*pudp))) { - put_compound_head(head, refs); + put_compound_head(head, refs, flags); return 0; } @@ -2203,12 +2507,12 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT); refs = record_subpages(page, addr, end, pages + *nr); - head = try_get_compound_head(pgd_page(orig), refs); + head = try_grab_compound_head(pgd_page(orig), refs, flags); if (!head) return 0; if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) { - put_compound_head(head, refs); + put_compound_head(head, refs, flags); return 0; } @@ -2370,7 +2674,15 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write, { unsigned long len, end; unsigned long flags; - int nr = 0; + int nr_pinned = 0; + /* + * Internally (within mm/gup.c), gup fast variants must set FOLL_GET, + * because gup fast is always a "pin with a +1 page refcount" request. + */ + unsigned int gup_flags = FOLL_GET; + + if (write) + gup_flags |= FOLL_WRITE; start = untagged_addr(start) & PAGE_MASK; len = (unsigned long) nr_pages << PAGE_SHIFT; @@ -2396,11 +2708,11 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write, if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) && gup_fast_permitted(start, end)) { local_irq_save(flags); - gup_pgd_range(start, end, write ? FOLL_WRITE : 0, pages, &nr); + gup_pgd_range(start, end, gup_flags, pages, &nr_pinned); local_irq_restore(flags); } - return nr; + return nr_pinned; } EXPORT_SYMBOL_GPL(__get_user_pages_fast); @@ -2432,10 +2744,10 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages, struct page **pages) { unsigned long addr, len, end; - int nr = 0, ret = 0; + int nr_pinned = 0, ret = 0; if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM | - FOLL_FORCE | FOLL_PIN))) + FOLL_FORCE | FOLL_PIN | FOLL_GET))) return -EINVAL; start = untagged_addr(start) & PAGE_MASK; @@ -2451,25 +2763,25 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages, if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) && gup_fast_permitted(start, end)) { local_irq_disable(); - gup_pgd_range(addr, end, gup_flags, pages, &nr); + gup_pgd_range(addr, end, gup_flags, pages, &nr_pinned); local_irq_enable(); - ret = nr; + ret = nr_pinned; } - if (nr < nr_pages) { + if (nr_pinned < nr_pages) { /* Try to get the remaining pages with get_user_pages */ - start += nr << PAGE_SHIFT; - pages += nr; + start += nr_pinned << PAGE_SHIFT; + pages += nr_pinned; - ret = __gup_longterm_unlocked(start, nr_pages - nr, + ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags, pages); /* Have to be a bit careful with return values */ - if (nr > 0) { + if (nr_pinned > 0) { if (ret < 0) - ret = nr; + ret = nr_pinned; else - ret += nr; + ret += nr_pinned; } } @@ -2478,11 +2790,11 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages, /** * get_user_pages_fast() - pin user pages in memory - * @start: starting user address - * @nr_pages: number of pages from start to pin - * @gup_flags: flags modifying pin behaviour - * @pages: array that receives pointers to the pages pinned. - * Should be at least nr_pages long. + * @start: starting user address + * @nr_pages: number of pages from start to pin + * @gup_flags: flags modifying pin behaviour + * @pages: array that receives pointers to the pages pinned. + * Should be at least nr_pages long. * * Attempt to pin user pages in memory without taking mm->mmap_sem. * If not successful, it will fall back to taking the lock and @@ -2502,6 +2814,13 @@ int get_user_pages_fast(unsigned long start, int nr_pages, if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) return -EINVAL; + /* + * The caller may or may not have explicitly set FOLL_GET; either way is + * OK. However, internally (within mm/gup.c), gup fast variants must set + * FOLL_GET, because gup fast is always a "pin with a +1 page refcount" + * request. + */ + gup_flags |= FOLL_GET; return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); } EXPORT_SYMBOL_GPL(get_user_pages_fast); @@ -2509,9 +2828,18 @@ EXPORT_SYMBOL_GPL(get_user_pages_fast); /** * pin_user_pages_fast() - pin user pages in memory without taking locks * - * For now, this is a placeholder function, until various call sites are - * converted to use the correct get_user_pages*() or pin_user_pages*() API. So, - * this is identical to get_user_pages_fast(). + * @start: starting user address + * @nr_pages: number of pages from start to pin + * @gup_flags: flags modifying pin behaviour + * @pages: array that receives pointers to the pages pinned. + * Should be at least nr_pages long. + * + * Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See + * get_user_pages_fast() for documentation on the function arguments, because + * the arguments here are identical. + * + * FOLL_PIN means that the pages must be released via unpin_user_page(). Please + * see Documentation/vm/pin_user_pages.rst for further details. * * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It * is NOT intended for Case 2 (RDMA: long-term pins). @@ -2519,21 +2847,39 @@ EXPORT_SYMBOL_GPL(get_user_pages_fast); int pin_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages) { - /* - * This is a placeholder, until the pin functionality is activated. - * Until then, just behave like the corresponding get_user_pages*() - * routine. - */ - return get_user_pages_fast(start, nr_pages, gup_flags, pages); + /* FOLL_GET and FOLL_PIN are mutually exclusive. */ + if (WARN_ON_ONCE(gup_flags & FOLL_GET)) + return -EINVAL; + + gup_flags |= FOLL_PIN; + return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); } EXPORT_SYMBOL_GPL(pin_user_pages_fast); /** * pin_user_pages_remote() - pin pages of a remote process (task != current) * - * For now, this is a placeholder function, until various call sites are - * converted to use the correct get_user_pages*() or pin_user_pages*() API. So, - * this is identical to get_user_pages_remote(). + * @tsk: the task_struct to use for page fault accounting, or + * NULL if faults are not to be recorded. + * @mm: mm_struct of target mm + * @start: starting user address + * @nr_pages: number of pages from start to pin + * @gup_flags: flags modifying lookup behaviour + * @pages: array that receives pointers to the pages pinned. + * Should be at least nr_pages long. Or NULL, if caller + * only intends to ensure the pages are faulted in. + * @vmas: array of pointers to vmas corresponding to each page. + * Or NULL if the caller does not require them. + * @locked: pointer to lock flag indicating whether lock is held and + * subsequently whether VM_FAULT_RETRY functionality can be + * utilised. Lock must initially be held. + * + * Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See + * get_user_pages_remote() for documentation on the function arguments, because + * the arguments here are identical. + * + * FOLL_PIN means that the pages must be released via unpin_user_page(). Please + * see Documentation/vm/pin_user_pages.rst for details. * * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It * is NOT intended for Case 2 (RDMA: long-term pins). @@ -2543,22 +2889,33 @@ long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas, int *locked) { - /* - * This is a placeholder, until the pin functionality is activated. - * Until then, just behave like the corresponding get_user_pages*() - * routine. - */ - return get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags, pages, - vmas, locked); + /* FOLL_GET and FOLL_PIN are mutually exclusive. */ + if (WARN_ON_ONCE(gup_flags & FOLL_GET)) + return -EINVAL; + + gup_flags |= FOLL_PIN; + return __get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags, + pages, vmas, locked); } EXPORT_SYMBOL(pin_user_pages_remote); /** * pin_user_pages() - pin user pages in memory for use by other devices * - * For now, this is a placeholder function, until various call sites are - * converted to use the correct get_user_pages*() or pin_user_pages*() API. So, - * this is identical to get_user_pages(). + * @start: starting user address + * @nr_pages: number of pages from start to pin + * @gup_flags: flags modifying lookup behaviour + * @pages: array that receives pointers to the pages pinned. + * Should be at least nr_pages long. Or NULL, if caller + * only intends to ensure the pages are faulted in. + * @vmas: array of pointers to vmas corresponding to each page. + * Or NULL if the caller does not require them. + * + * Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and + * FOLL_PIN is set. + * + * FOLL_PIN means that the pages must be released via unpin_user_page(). Please + * see Documentation/vm/pin_user_pages.rst for details. * * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It * is NOT intended for Case 2 (RDMA: long-term pins). @@ -2567,11 +2924,12 @@ long pin_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas) { - /* - * This is a placeholder, until the pin functionality is activated. - * Until then, just behave like the corresponding get_user_pages*() - * routine. - */ - return get_user_pages(start, nr_pages, gup_flags, pages, vmas); + /* FOLL_GET and FOLL_PIN are mutually exclusive. */ + if (WARN_ON_ONCE(gup_flags & FOLL_GET)) + return -EINVAL; + + gup_flags |= FOLL_PIN; + return __gup_longterm_locked(current, current->mm, start, nr_pages, + pages, vmas, gup_flags); } EXPORT_SYMBOL(pin_user_pages); diff --git a/mm/gup_benchmark.c b/mm/gup_benchmark.c index 8dba38e79a9f..be690fa66a46 100644 --- a/mm/gup_benchmark.c +++ b/mm/gup_benchmark.c @@ -8,6 +8,8 @@ #define GUP_FAST_BENCHMARK _IOWR('g', 1, struct gup_benchmark) #define GUP_LONGTERM_BENCHMARK _IOWR('g', 2, struct gup_benchmark) #define GUP_BENCHMARK _IOWR('g', 3, struct gup_benchmark) +#define PIN_FAST_BENCHMARK _IOWR('g', 4, struct gup_benchmark) +#define PIN_BENCHMARK _IOWR('g', 5, struct gup_benchmark) struct gup_benchmark { __u64 get_delta_usec; @@ -19,6 +21,48 @@ struct gup_benchmark { __u64 expansion[10]; /* For future use */ }; +static void put_back_pages(unsigned int cmd, struct page **pages, + unsigned long nr_pages) +{ + unsigned long i; + + switch (cmd) { + case GUP_FAST_BENCHMARK: + case GUP_LONGTERM_BENCHMARK: + case GUP_BENCHMARK: + for (i = 0; i < nr_pages; i++) + put_page(pages[i]); + break; + + case PIN_FAST_BENCHMARK: + case PIN_BENCHMARK: + unpin_user_pages(pages, nr_pages); + break; + } +} + +static void verify_dma_pinned(unsigned int cmd, struct page **pages, + unsigned long nr_pages) +{ + unsigned long i; + struct page *page; + + switch (cmd) { + case PIN_FAST_BENCHMARK: + case PIN_BENCHMARK: + for (i = 0; i < nr_pages; i++) { + page = pages[i]; + if (WARN(!page_maybe_dma_pinned(page), + "pages[%lu] is NOT dma-pinned\n", i)) { + + dump_page(page, "gup_benchmark failure"); + break; + } + } + break; + } +} + static int __gup_benchmark_ioctl(unsigned int cmd, struct gup_benchmark *gup) { @@ -66,6 +110,14 @@ static int __gup_benchmark_ioctl(unsigned int cmd, nr = get_user_pages(addr, nr, gup->flags, pages + i, NULL); break; + case PIN_FAST_BENCHMARK: + nr = pin_user_pages_fast(addr, nr, gup->flags, + pages + i); + break; + case PIN_BENCHMARK: + nr = pin_user_pages(addr, nr, gup->flags, pages + i, + NULL); + break; default: kvfree(pages); ret = -EINVAL; @@ -78,15 +130,22 @@ static int __gup_benchmark_ioctl(unsigned int cmd, } end_time = ktime_get(); + /* Shifting the meaning of nr_pages: now it is actual number pinned: */ + nr_pages = i; + gup->get_delta_usec = ktime_us_delta(end_time, start_time); gup->size = addr - gup->addr; + /* + * Take an un-benchmark-timed moment to verify DMA pinned + * state: print a warning if any non-dma-pinned pages are found: + */ + verify_dma_pinned(cmd, pages, nr_pages); + start_time = ktime_get(); - for (i = 0; i < nr_pages; i++) { - if (!pages[i]) - break; - put_page(pages[i]); - } + + put_back_pages(cmd, pages, nr_pages); + end_time = ktime_get(); gup->put_delta_usec = ktime_us_delta(end_time, start_time); @@ -105,6 +164,8 @@ static long gup_benchmark_ioctl(struct file *filep, unsigned int cmd, case GUP_FAST_BENCHMARK: case GUP_LONGTERM_BENCHMARK: case GUP_BENCHMARK: + case PIN_FAST_BENCHMARK: + case PIN_BENCHMARK: break; default: return -EINVAL; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 24ad53b4dfc0..b1e069e68189 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -958,6 +958,11 @@ struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, */ WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set"); + /* FOLL_GET and FOLL_PIN are mutually exclusive. */ + if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == + (FOLL_PIN | FOLL_GET))) + return NULL; + if (flags & FOLL_WRITE && !pmd_write(*pmd)) return NULL; @@ -973,7 +978,7 @@ struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, * device mapped pages can only be returned if the * caller will manage the page reference count. */ - if (!(flags & FOLL_GET)) + if (!(flags & (FOLL_GET | FOLL_PIN))) return ERR_PTR(-EEXIST); pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; @@ -981,7 +986,8 @@ struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, if (!*pgmap) return ERR_PTR(-EFAULT); page = pfn_to_page(pfn); - get_page(page); + if (!try_grab_page(page, flags)) + page = ERR_PTR(-ENOMEM); return page; } @@ -1101,6 +1107,11 @@ struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, if (flags & FOLL_WRITE && !pud_write(*pud)) return NULL; + /* FOLL_GET and FOLL_PIN are mutually exclusive. */ + if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == + (FOLL_PIN | FOLL_GET))) + return NULL; + if (pud_present(*pud) && pud_devmap(*pud)) /* pass */; else @@ -1112,8 +1123,10 @@ struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, /* * device mapped pages can only be returned if the * caller will manage the page reference count. + * + * At least one of FOLL_GET | FOLL_PIN must be set, so assert that here: */ - if (!(flags & FOLL_GET)) + if (!(flags & (FOLL_GET | FOLL_PIN))) return ERR_PTR(-EEXIST); pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT; @@ -1121,7 +1134,8 @@ struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, if (!*pgmap) return ERR_PTR(-EFAULT); page = pfn_to_page(pfn); - get_page(page); + if (!try_grab_page(page, flags)) + page = ERR_PTR(-ENOMEM); return page; } @@ -1497,8 +1511,13 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, page = pmd_page(*pmd); VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page); + + if (!try_grab_page(page, flags)) + return ERR_PTR(-ENOMEM); + if (flags & FOLL_TOUCH) touch_pmd(vma, addr, pmd, flags); + if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { /* * We don't mlock() pte-mapped THPs. This way we can avoid @@ -1535,8 +1554,6 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, skip_mlock: page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page); - if (flags & FOLL_GET) - get_page(page); out: return page; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index dd8737a94bec..f9ea1e5197b4 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -220,132 +220,303 @@ static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) return subpool_inode(file_inode(vma->vm_file)); } -/* - * Region tracking -- allows tracking of reservations and instantiated pages - * across the pages in a mapping. - * - * The region data structures are embedded into a resv_map and protected - * by a resv_map's lock. The set of regions within the resv_map represent - * reservations for huge pages, or huge pages that have already been - * instantiated within the map. The from and to elements are huge page - * indicies into the associated mapping. from indicates the starting index - * of the region. to represents the first index past the end of the region. - * - * For example, a file region structure with from == 0 and to == 4 represents - * four huge pages in a mapping. It is important to note that the to element - * represents the first element past the end of the region. This is used in - * arithmetic as 4(to) - 0(from) = 4 huge pages in the region. - * - * Interval notation of the form [from, to) will be used to indicate that - * the endpoint from is inclusive and to is exclusive. +/* Helper that removes a struct file_region from the resv_map cache and returns + * it for use. */ -struct file_region { - struct list_head link; - long from; - long to; -}; +static struct file_region * +get_file_region_entry_from_cache(struct resv_map *resv, long from, long to) +{ + struct file_region *nrg = NULL; + + VM_BUG_ON(resv->region_cache_count <= 0); + + resv->region_cache_count--; + nrg = list_first_entry(&resv->region_cache, struct file_region, link); + VM_BUG_ON(!nrg); + list_del(&nrg->link); + + nrg->from = from; + nrg->to = to; + + return nrg; +} + +static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg, + struct file_region *rg) +{ +#ifdef CONFIG_CGROUP_HUGETLB + nrg->reservation_counter = rg->reservation_counter; + nrg->css = rg->css; + if (rg->css) + css_get(rg->css); +#endif +} + +/* Helper that records hugetlb_cgroup uncharge info. */ +static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg, + struct hstate *h, + struct resv_map *resv, + struct file_region *nrg) +{ +#ifdef CONFIG_CGROUP_HUGETLB + if (h_cg) { + nrg->reservation_counter = + &h_cg->rsvd_hugepage[hstate_index(h)]; + nrg->css = &h_cg->css; + if (!resv->pages_per_hpage) + resv->pages_per_hpage = pages_per_huge_page(h); + /* pages_per_hpage should be the same for all entries in + * a resv_map. + */ + VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h)); + } else { + nrg->reservation_counter = NULL; + nrg->css = NULL; + } +#endif +} + +static bool has_same_uncharge_info(struct file_region *rg, + struct file_region *org) +{ +#ifdef CONFIG_CGROUP_HUGETLB + return rg && org && + rg->reservation_counter == org->reservation_counter && + rg->css == org->css; + +#else + return true; +#endif +} + +static void coalesce_file_region(struct resv_map *resv, struct file_region *rg) +{ + struct file_region *nrg = NULL, *prg = NULL; + + prg = list_prev_entry(rg, link); + if (&prg->link != &resv->regions && prg->to == rg->from && + has_same_uncharge_info(prg, rg)) { + prg->to = rg->to; + + list_del(&rg->link); + kfree(rg); + + coalesce_file_region(resv, prg); + return; + } + + nrg = list_next_entry(rg, link); + if (&nrg->link != &resv->regions && nrg->from == rg->to && + has_same_uncharge_info(nrg, rg)) { + nrg->from = rg->from; + + list_del(&rg->link); + kfree(rg); + + coalesce_file_region(resv, nrg); + return; + } +} /* Must be called with resv->lock held. Calling this with count_only == true * will count the number of pages to be added but will not modify the linked - * list. + * list. If regions_needed != NULL and count_only == true, then regions_needed + * will indicate the number of file_regions needed in the cache to carry out to + * add the regions for this range. */ static long add_reservation_in_range(struct resv_map *resv, long f, long t, + struct hugetlb_cgroup *h_cg, + struct hstate *h, long *regions_needed, bool count_only) { - long chg = 0; + long add = 0; struct list_head *head = &resv->regions; + long last_accounted_offset = f; struct file_region *rg = NULL, *trg = NULL, *nrg = NULL; - /* Locate the region we are before or in. */ - list_for_each_entry(rg, head, link) - if (f <= rg->to) - break; + if (regions_needed) + *regions_needed = 0; - /* Round our left edge to the current segment if it encloses us. */ - if (f > rg->from) - f = rg->from; + /* In this loop, we essentially handle an entry for the range + * [last_accounted_offset, rg->from), at every iteration, with some + * bounds checking. + */ + list_for_each_entry_safe(rg, trg, head, link) { + /* Skip irrelevant regions that start before our range. */ + if (rg->from < f) { + /* If this region ends after the last accounted offset, + * then we need to update last_accounted_offset. + */ + if (rg->to > last_accounted_offset) + last_accounted_offset = rg->to; + continue; + } - chg = t - f; - - /* Check for and consume any regions we now overlap with. */ - nrg = rg; - list_for_each_entry_safe(rg, trg, rg->link.prev, link) { - if (&rg->link == head) - break; + /* When we find a region that starts beyond our range, we've + * finished. + */ if (rg->from > t) break; - /* We overlap with this area, if it extends further than - * us then we must extend ourselves. Account for its - * existing reservation. + /* Add an entry for last_accounted_offset -> rg->from, and + * update last_accounted_offset. */ - if (rg->to > t) { - chg += rg->to - t; - t = rg->to; + if (rg->from > last_accounted_offset) { + add += rg->from - last_accounted_offset; + if (!count_only) { + nrg = get_file_region_entry_from_cache( + resv, last_accounted_offset, rg->from); + record_hugetlb_cgroup_uncharge_info(h_cg, h, + resv, nrg); + list_add(&nrg->link, rg->link.prev); + coalesce_file_region(resv, nrg); + } else if (regions_needed) + *regions_needed += 1; } - chg -= rg->to - rg->from; - if (!count_only && rg != nrg) { + last_accounted_offset = rg->to; + } + + /* Handle the case where our range extends beyond + * last_accounted_offset. + */ + if (last_accounted_offset < t) { + add += t - last_accounted_offset; + if (!count_only) { + nrg = get_file_region_entry_from_cache( + resv, last_accounted_offset, t); + record_hugetlb_cgroup_uncharge_info(h_cg, h, resv, nrg); + list_add(&nrg->link, rg->link.prev); + coalesce_file_region(resv, nrg); + } else if (regions_needed) + *regions_needed += 1; + } + + VM_BUG_ON(add < 0); + return add; +} + +/* Must be called with resv->lock acquired. Will drop lock to allocate entries. + */ +static int allocate_file_region_entries(struct resv_map *resv, + int regions_needed) + __must_hold(&resv->lock) +{ + struct list_head allocated_regions; + int to_allocate = 0, i = 0; + struct file_region *trg = NULL, *rg = NULL; + + VM_BUG_ON(regions_needed < 0); + + INIT_LIST_HEAD(&allocated_regions); + + /* + * Check for sufficient descriptors in the cache to accommodate + * the number of in progress add operations plus regions_needed. + * + * This is a while loop because when we drop the lock, some other call + * to region_add or region_del may have consumed some region_entries, + * so we keep looping here until we finally have enough entries for + * (adds_in_progress + regions_needed). + */ + while (resv->region_cache_count < + (resv->adds_in_progress + regions_needed)) { + to_allocate = resv->adds_in_progress + regions_needed - + resv->region_cache_count; + + /* At this point, we should have enough entries in the cache + * for all the existings adds_in_progress. We should only be + * needing to allocate for regions_needed. + */ + VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress); + + spin_unlock(&resv->lock); + for (i = 0; i < to_allocate; i++) { + trg = kmalloc(sizeof(*trg), GFP_KERNEL); + if (!trg) + goto out_of_memory; + list_add(&trg->link, &allocated_regions); + } + + spin_lock(&resv->lock); + + list_for_each_entry_safe(rg, trg, &allocated_regions, link) { list_del(&rg->link); - kfree(rg); + list_add(&rg->link, &resv->region_cache); + resv->region_cache_count++; } } - if (!count_only) { - nrg->from = f; - nrg->to = t; - } + return 0; - return chg; +out_of_memory: + list_for_each_entry_safe(rg, trg, &allocated_regions, link) { + list_del(&rg->link); + kfree(rg); + } + return -ENOMEM; } /* * Add the huge page range represented by [f, t) to the reserve - * map. Existing regions will be expanded to accommodate the specified - * range, or a region will be taken from the cache. Sufficient regions - * must exist in the cache due to the previous call to region_chg with - * the same range. + * map. Regions will be taken from the cache to fill in this range. + * Sufficient regions should exist in the cache due to the previous + * call to region_chg with the same range, but in some cases the cache will not + * have sufficient entries due to races with other code doing region_add or + * region_del. The extra needed entries will be allocated. * - * Return the number of new huge pages added to the map. This - * number is greater than or equal to zero. + * regions_needed is the out value provided by a previous call to region_chg. + * + * Return the number of new huge pages added to the map. This number is greater + * than or equal to zero. If file_region entries needed to be allocated for + * this operation and we were not able to allocate, it ruturns -ENOMEM. + * region_add of regions of length 1 never allocate file_regions and cannot + * fail; region_chg will always allocate at least 1 entry and a region_add for + * 1 page will only require at most 1 entry. */ -static long region_add(struct resv_map *resv, long f, long t) +static long region_add(struct resv_map *resv, long f, long t, + long in_regions_needed, struct hstate *h, + struct hugetlb_cgroup *h_cg) { - struct list_head *head = &resv->regions; - struct file_region *rg, *nrg; - long add = 0; + long add = 0, actual_regions_needed = 0; spin_lock(&resv->lock); - /* Locate the region we are either in or before. */ - list_for_each_entry(rg, head, link) - if (f <= rg->to) - break; +retry: + + /* Count how many regions are actually needed to execute this add. */ + add_reservation_in_range(resv, f, t, NULL, NULL, &actual_regions_needed, + true); /* - * If no region exists which can be expanded to include the - * specified range, pull a region descriptor from the cache - * and use it for this range. + * Check for sufficient descriptors in the cache to accommodate + * this add operation. Note that actual_regions_needed may be greater + * than in_regions_needed, as the resv_map may have been modified since + * the region_chg call. In this case, we need to make sure that we + * allocate extra entries, such that we have enough for all the + * existing adds_in_progress, plus the excess needed for this + * operation. */ - if (&rg->link == head || t < rg->from) { - VM_BUG_ON(resv->region_cache_count <= 0); + if (actual_regions_needed > in_regions_needed && + resv->region_cache_count < + resv->adds_in_progress + + (actual_regions_needed - in_regions_needed)) { + /* region_add operation of range 1 should never need to + * allocate file_region entries. + */ + VM_BUG_ON(t - f <= 1); - resv->region_cache_count--; - nrg = list_first_entry(&resv->region_cache, struct file_region, - link); - list_del(&nrg->link); + if (allocate_file_region_entries( + resv, actual_regions_needed - in_regions_needed)) { + return -ENOMEM; + } - nrg->from = f; - nrg->to = t; - list_add(&nrg->link, rg->link.prev); - - add += t - f; - goto out_locked; + goto retry; } - add = add_reservation_in_range(resv, f, t, false); + add = add_reservation_in_range(resv, f, t, h_cg, h, NULL, false); + + resv->adds_in_progress -= in_regions_needed; -out_locked: - resv->adds_in_progress--; spin_unlock(&resv->lock); VM_BUG_ON(add < 0); return add; @@ -358,46 +529,37 @@ static long region_add(struct resv_map *resv, long f, long t) * call to region_add that will actually modify the reserve * map to add the specified range [f, t). region_chg does * not change the number of huge pages represented by the - * map. A new file_region structure is added to the cache - * as a placeholder, so that the subsequent region_add - * call will have all the regions it needs and will not fail. + * map. A number of new file_region structures is added to the cache as a + * placeholder, for the subsequent region_add call to use. At least 1 + * file_region structure is added. + * + * out_regions_needed is the number of regions added to the + * resv->adds_in_progress. This value needs to be provided to a follow up call + * to region_add or region_abort for proper accounting. * * Returns the number of huge pages that need to be added to the existing * reservation map for the range [f, t). This number is greater or equal to * zero. -ENOMEM is returned if a new file_region structure or cache entry * is needed and can not be allocated. */ -static long region_chg(struct resv_map *resv, long f, long t) +static long region_chg(struct resv_map *resv, long f, long t, + long *out_regions_needed) { long chg = 0; spin_lock(&resv->lock); -retry_locked: - resv->adds_in_progress++; - /* - * Check for sufficient descriptors in the cache to accommodate - * the number of in progress add operations. - */ - if (resv->adds_in_progress > resv->region_cache_count) { - struct file_region *trg; + /* Count how many hugepages in this range are NOT respresented. */ + chg = add_reservation_in_range(resv, f, t, NULL, NULL, + out_regions_needed, true); - VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1); - /* Must drop lock to allocate a new descriptor. */ - resv->adds_in_progress--; - spin_unlock(&resv->lock); + if (*out_regions_needed == 0) + *out_regions_needed = 1; - trg = kmalloc(sizeof(*trg), GFP_KERNEL); - if (!trg) - return -ENOMEM; + if (allocate_file_region_entries(resv, *out_regions_needed)) + return -ENOMEM; - spin_lock(&resv->lock); - list_add(&trg->link, &resv->region_cache); - resv->region_cache_count++; - goto retry_locked; - } - - chg = add_reservation_in_range(resv, f, t, true); + resv->adds_in_progress += *out_regions_needed; spin_unlock(&resv->lock); return chg; @@ -408,17 +570,20 @@ static long region_chg(struct resv_map *resv, long f, long t) * of the resv_map keeps track of the operations in progress between * calls to region_chg and region_add. Operations are sometimes * aborted after the call to region_chg. In such cases, region_abort - * is called to decrement the adds_in_progress counter. + * is called to decrement the adds_in_progress counter. regions_needed + * is the value returned by the region_chg call, it is used to decrement + * the adds_in_progress counter. * * NOTE: The range arguments [f, t) are not needed or used in this * routine. They are kept to make reading the calling code easier as * arguments will match the associated region_chg call. */ -static void region_abort(struct resv_map *resv, long f, long t) +static void region_abort(struct resv_map *resv, long f, long t, + long regions_needed) { spin_lock(&resv->lock); VM_BUG_ON(!resv->region_cache_count); - resv->adds_in_progress--; + resv->adds_in_progress -= regions_needed; spin_unlock(&resv->lock); } @@ -486,11 +651,17 @@ static long region_del(struct resv_map *resv, long f, long t) /* New entry for end of split region */ nrg->from = t; nrg->to = rg->to; + + copy_hugetlb_cgroup_uncharge_info(nrg, rg); + INIT_LIST_HEAD(&nrg->link); /* Original entry is trimmed */ rg->to = f; + hugetlb_cgroup_uncharge_file_region( + resv, rg, nrg->to - nrg->from); + list_add(&nrg->link, &rg->link); nrg = NULL; break; @@ -498,6 +669,8 @@ static long region_del(struct resv_map *resv, long f, long t) if (f <= rg->from && t >= rg->to) { /* Remove entire region */ del += rg->to - rg->from; + hugetlb_cgroup_uncharge_file_region(resv, rg, + rg->to - rg->from); list_del(&rg->link); kfree(rg); continue; @@ -506,9 +679,15 @@ static long region_del(struct resv_map *resv, long f, long t) if (f <= rg->from) { /* Trim beginning of region */ del += t - rg->from; rg->from = t; + + hugetlb_cgroup_uncharge_file_region(resv, rg, + t - rg->from); } else { /* Trim end of region */ del += rg->to - f; rg->to = f; + + hugetlb_cgroup_uncharge_file_region(resv, rg, + rg->to - f); } } @@ -650,6 +829,25 @@ static void set_vma_private_data(struct vm_area_struct *vma, vma->vm_private_data = (void *)value; } +static void +resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map, + struct hugetlb_cgroup *h_cg, + struct hstate *h) +{ +#ifdef CONFIG_CGROUP_HUGETLB + if (!h_cg || !h) { + resv_map->reservation_counter = NULL; + resv_map->pages_per_hpage = 0; + resv_map->css = NULL; + } else { + resv_map->reservation_counter = + &h_cg->rsvd_hugepage[hstate_index(h)]; + resv_map->pages_per_hpage = pages_per_huge_page(h); + resv_map->css = &h_cg->css; + } +#endif +} + struct resv_map *resv_map_alloc(void) { struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL); @@ -666,6 +864,13 @@ struct resv_map *resv_map_alloc(void) INIT_LIST_HEAD(&resv_map->regions); resv_map->adds_in_progress = 0; + /* + * Initialize these to 0. On shared mappings, 0's here indicate these + * fields don't do cgroup accounting. On private mappings, these will be + * re-initialized to the proper values, to indicate that hugetlb cgroup + * reservations are to be un-charged from here. + */ + resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL); INIT_LIST_HEAD(&resv_map->region_cache); list_add(&rg->link, &resv_map->region_cache); @@ -1009,6 +1214,9 @@ static void destroy_compound_gigantic_page(struct page *page, struct page *p = page + 1; atomic_set(compound_mapcount_ptr(page), 0); + if (hpage_pincount_available(page)) + atomic_set(compound_pincount_ptr(page), 0); + for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { clear_compound_head(p); set_page_refcounted(p); @@ -1069,6 +1277,7 @@ static void update_and_free_page(struct hstate *h, struct page *page) 1 << PG_writeback); } VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page); + VM_BUG_ON_PAGE(hugetlb_cgroup_from_page_rsvd(page), page); set_compound_page_dtor(page, NULL_COMPOUND_DTOR); set_page_refcounted(page); if (hstate_is_gigantic(h)) { @@ -1180,6 +1389,8 @@ static void __free_huge_page(struct page *page) clear_page_huge_active(page); hugetlb_cgroup_uncharge_page(hstate_index(h), pages_per_huge_page(h), page); + hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h), + pages_per_huge_page(h), page); if (restore_reserve) h->resv_huge_pages++; @@ -1254,6 +1465,7 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); spin_lock(&hugetlb_lock); set_hugetlb_cgroup(page, NULL); + set_hugetlb_cgroup_rsvd(page, NULL); h->nr_huge_pages++; h->nr_huge_pages_node[nid]++; spin_unlock(&hugetlb_lock); @@ -1287,6 +1499,9 @@ static void prep_compound_gigantic_page(struct page *page, unsigned int order) set_compound_head(p, page); } atomic_set(compound_mapcount_ptr(page), -1); + + if (hpage_pincount_available(page)) + atomic_set(compound_pincount_ptr(page), 0); } /* @@ -1313,7 +1528,107 @@ int PageHeadHuge(struct page *page_head) if (!PageHead(page_head)) return 0; - return get_compound_page_dtor(page_head) == free_huge_page; + return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR; +} + +/* + * Find address_space associated with hugetlbfs page. + * Upon entry page is locked and page 'was' mapped although mapped state + * could change. If necessary, use anon_vma to find vma and associated + * address space. The returned mapping may be stale, but it can not be + * invalid as page lock (which is held) is required to destroy mapping. + */ +static struct address_space *_get_hugetlb_page_mapping(struct page *hpage) +{ + struct anon_vma *anon_vma; + pgoff_t pgoff_start, pgoff_end; + struct anon_vma_chain *avc; + struct address_space *mapping = page_mapping(hpage); + + /* Simple file based mapping */ + if (mapping) + return mapping; + + /* + * Even anonymous hugetlbfs mappings are associated with an + * underlying hugetlbfs file (see hugetlb_file_setup in mmap + * code). Find a vma associated with the anonymous vma, and + * use the file pointer to get address_space. + */ + anon_vma = page_lock_anon_vma_read(hpage); + if (!anon_vma) + return mapping; /* NULL */ + + /* Use first found vma */ + pgoff_start = page_to_pgoff(hpage); + pgoff_end = pgoff_start + hpage_nr_pages(hpage) - 1; + anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, + pgoff_start, pgoff_end) { + struct vm_area_struct *vma = avc->vma; + + mapping = vma->vm_file->f_mapping; + break; + } + + anon_vma_unlock_read(anon_vma); + return mapping; +} + +/* + * Find and lock address space (mapping) in write mode. + * + * Upon entry, the page is locked which allows us to find the mapping + * even in the case of an anon page. However, locking order dictates + * the i_mmap_rwsem be acquired BEFORE the page lock. This is hugetlbfs + * specific. So, we first try to lock the sema while still holding the + * page lock. If this works, great! If not, then we need to drop the + * page lock and then acquire i_mmap_rwsem and reacquire page lock. Of + * course, need to revalidate state along the way. + */ +struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage) +{ + struct address_space *mapping, *mapping2; + + mapping = _get_hugetlb_page_mapping(hpage); +retry: + if (!mapping) + return mapping; + + /* + * If no contention, take lock and return + */ + if (i_mmap_trylock_write(mapping)) + return mapping; + + /* + * Must drop page lock and wait on mapping sema. + * Note: Once page lock is dropped, mapping could become invalid. + * As a hack, increase map count until we lock page again. + */ + atomic_inc(&hpage->_mapcount); + unlock_page(hpage); + i_mmap_lock_write(mapping); + lock_page(hpage); + atomic_add_negative(-1, &hpage->_mapcount); + + /* verify page is still mapped */ + if (!page_mapped(hpage)) { + i_mmap_unlock_write(mapping); + return NULL; + } + + /* + * Get address space again and verify it is the same one + * we locked. If not, drop lock and retry. + */ + mapping2 = _get_hugetlb_page_mapping(hpage); + if (mapping2 != mapping) { + i_mmap_unlock_write(mapping); + mapping = mapping2; + goto retry; + } + + return mapping; } pgoff_t __basepage_index(struct page *page) @@ -1870,6 +2185,7 @@ static long __vma_reservation_common(struct hstate *h, struct resv_map *resv; pgoff_t idx; long ret; + long dummy_out_regions_needed; resv = vma_resv_map(vma); if (!resv) @@ -1878,20 +2194,29 @@ static long __vma_reservation_common(struct hstate *h, idx = vma_hugecache_offset(h, vma, addr); switch (mode) { case VMA_NEEDS_RESV: - ret = region_chg(resv, idx, idx + 1); + ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed); + /* We assume that vma_reservation_* routines always operate on + * 1 page, and that adding to resv map a 1 page entry can only + * ever require 1 region. + */ + VM_BUG_ON(dummy_out_regions_needed != 1); break; case VMA_COMMIT_RESV: - ret = region_add(resv, idx, idx + 1); + ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); + /* region_add calls of range 1 should never fail. */ + VM_BUG_ON(ret < 0); break; case VMA_END_RESV: - region_abort(resv, idx, idx + 1); + region_abort(resv, idx, idx + 1, 1); ret = 0; break; case VMA_ADD_RESV: - if (vma->vm_flags & VM_MAYSHARE) - ret = region_add(resv, idx, idx + 1); - else { - region_abort(resv, idx, idx + 1); + if (vma->vm_flags & VM_MAYSHARE) { + ret = region_add(resv, idx, idx + 1, 1, NULL, NULL); + /* region_add calls of range 1 should never fail. */ + VM_BUG_ON(ret < 0); + } else { + region_abort(resv, idx, idx + 1, 1); ret = region_del(resv, idx, idx + 1); } break; @@ -2002,6 +2327,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, long gbl_chg; int ret, idx; struct hugetlb_cgroup *h_cg; + bool deferred_reserve; idx = hstate_index(h); /* @@ -2039,9 +2365,19 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, gbl_chg = 1; } + /* If this allocation is not consuming a reservation, charge it now. + */ + deferred_reserve = map_chg || avoid_reserve || !vma_resv_map(vma); + if (deferred_reserve) { + ret = hugetlb_cgroup_charge_cgroup_rsvd( + idx, pages_per_huge_page(h), &h_cg); + if (ret) + goto out_subpool_put; + } + ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); if (ret) - goto out_subpool_put; + goto out_uncharge_cgroup_reservation; spin_lock(&hugetlb_lock); /* @@ -2064,6 +2400,14 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, /* Fall through */ } hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); + /* If allocation is not consuming a reservation, also store the + * hugetlb_cgroup pointer on the page. + */ + if (deferred_reserve) { + hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), + h_cg, page); + } + spin_unlock(&hugetlb_lock); set_page_private(page, (unsigned long)spool); @@ -2088,6 +2432,10 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, out_uncharge_cgroup: hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); +out_uncharge_cgroup_reservation: + if (deferred_reserve) + hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), + h_cg); out_subpool_put: if (map_chg || avoid_reserve) hugepage_subpool_put_pages(spool, 1); @@ -3188,9 +3536,7 @@ static void hugetlb_vm_op_close(struct vm_area_struct *vma) end = vma_hugecache_offset(h, vma, vma->vm_end); reserve = (end - start) - region_count(resv, start, end); - - kref_put(&resv->refs, resv_map_release); - + hugetlb_cgroup_uncharge_counter(resv, start, end); if (reserve) { /* * Decrement reserve counts. The global reserve count may be @@ -3199,6 +3545,8 @@ static void hugetlb_vm_op_close(struct vm_area_struct *vma) gbl_reserve = hugepage_subpool_put_pages(spool, reserve); hugetlb_acct_memory(h, -gbl_reserve); } + + kref_put(&resv->refs, resv_map_release); } static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr) @@ -3306,6 +3654,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, int cow; struct hstate *h = hstate_vma(vma); unsigned long sz = huge_page_size(h); + struct address_space *mapping = vma->vm_file->f_mapping; struct mmu_notifier_range range; int ret = 0; @@ -3316,6 +3665,14 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, vma->vm_start, vma->vm_end); mmu_notifier_invalidate_range_start(&range); + } else { + /* + * For shared mappings i_mmap_rwsem must be held to call + * huge_pte_alloc, otherwise the returned ptep could go + * away if part of a shared pmd and another thread calls + * huge_pmd_unshare. + */ + i_mmap_lock_read(mapping); } for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { @@ -3393,6 +3750,8 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, if (cow) mmu_notifier_invalidate_range_end(&range); + else + i_mmap_unlock_read(mapping); return ret; } @@ -3812,16 +4171,17 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, } /* - * Use page lock to guard against racing truncation - * before we get page_table_lock. + * We can not race with truncation due to holding i_mmap_rwsem. + * i_size is modified when holding i_mmap_rwsem, so check here + * once for faults beyond end of file. */ + size = i_size_read(mapping->host) >> huge_page_shift(h); + if (idx >= size) + goto out; + retry: page = find_lock_page(mapping, idx); if (!page) { - size = i_size_read(mapping->host) >> huge_page_shift(h); - if (idx >= size) - goto out; - /* * Check for page in userfault range */ @@ -3841,13 +4201,15 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, }; /* - * hugetlb_fault_mutex must be dropped before - * handling userfault. Reacquire after handling - * fault to make calling code simpler. + * hugetlb_fault_mutex and i_mmap_rwsem must be + * dropped before handling userfault. Reacquire + * after handling fault to make calling code simpler. */ hash = hugetlb_fault_mutex_hash(mapping, idx); mutex_unlock(&hugetlb_fault_mutex_table[hash]); + i_mmap_unlock_read(mapping); ret = handle_userfault(&vmf, VM_UFFD_MISSING); + i_mmap_lock_read(mapping); mutex_lock(&hugetlb_fault_mutex_table[hash]); goto out; } @@ -3925,10 +4287,6 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, } ptl = huge_pte_lock(h, mm, ptep); - size = i_size_read(mapping->host) >> huge_page_shift(h); - if (idx >= size) - goto backout; - ret = 0; if (!huge_pte_none(huge_ptep_get(ptep))) goto backout; @@ -4012,6 +4370,11 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); if (ptep) { + /* + * Since we hold no locks, ptep could be stale. That is + * OK as we are only making decisions based on content and + * not actually modifying content here. + */ entry = huge_ptep_get(ptep); if (unlikely(is_hugetlb_entry_migration(entry))) { migration_entry_wait_huge(vma, mm, ptep); @@ -4025,14 +4388,31 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, return VM_FAULT_OOM; } + /* + * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold + * until finished with ptep. This serves two purposes: + * 1) It prevents huge_pmd_unshare from being called elsewhere + * and making the ptep no longer valid. + * 2) It synchronizes us with i_size modifications during truncation. + * + * ptep could have already be assigned via huge_pte_offset. That + * is OK, as huge_pte_alloc will return the same value unless + * something has changed. + */ mapping = vma->vm_file->f_mapping; - idx = vma_hugecache_offset(h, vma, haddr); + i_mmap_lock_read(mapping); + ptep = huge_pte_alloc(mm, haddr, huge_page_size(h)); + if (!ptep) { + i_mmap_unlock_read(mapping); + return VM_FAULT_OOM; + } /* * Serialize hugepage allocation and instantiation, so that we don't * get spurious allocation failures if two CPUs race to instantiate * the same page in the page cache. */ + idx = vma_hugecache_offset(h, vma, haddr); hash = hugetlb_fault_mutex_hash(mapping, idx); mutex_lock(&hugetlb_fault_mutex_table[hash]); @@ -4120,6 +4500,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, } out_mutex: mutex_unlock(&hugetlb_fault_mutex_table[hash]); + i_mmap_unlock_read(mapping); /* * Generally it's safe to hold refcount during waiting page lock. But * here we just wait to defer the next page fault to avoid busy loop and @@ -4266,7 +4647,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, struct page **pages, struct vm_area_struct **vmas, unsigned long *position, unsigned long *nr_pages, - long i, unsigned int flags, int *nonblocking) + long i, unsigned int flags, int *locked) { unsigned long pfn_offset; unsigned long vaddr = *position; @@ -4337,14 +4718,17 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, spin_unlock(ptl); if (flags & FOLL_WRITE) fault_flags |= FAULT_FLAG_WRITE; - if (nonblocking) - fault_flags |= FAULT_FLAG_ALLOW_RETRY; + if (locked) + fault_flags |= FAULT_FLAG_ALLOW_RETRY | + FAULT_FLAG_KILLABLE; if (flags & FOLL_NOWAIT) fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT; if (flags & FOLL_TRIED) { - VM_WARN_ON_ONCE(fault_flags & - FAULT_FLAG_ALLOW_RETRY); + /* + * Note: FAULT_FLAG_ALLOW_RETRY and + * FAULT_FLAG_TRIED can co-exist + */ fault_flags |= FAULT_FLAG_TRIED; } ret = hugetlb_fault(mm, vma, vaddr, fault_flags); @@ -4354,9 +4738,9 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, break; } if (ret & VM_FAULT_RETRY) { - if (nonblocking && + if (locked && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT)) - *nonblocking = 0; + *locked = 0; *nr_pages = 0; /* * VM_FAULT_RETRY must not return an @@ -4375,19 +4759,6 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT; page = pte_page(huge_ptep_get(pte)); - /* - * Instead of doing 'try_get_page()' below in the same_page - * loop, just check the count once here. - */ - if (unlikely(page_count(page) <= 0)) { - if (pages) { - spin_unlock(ptl); - remainder = 0; - err = -ENOMEM; - break; - } - } - /* * If subpage information not requested, update counters * and skip the same_page loop below. @@ -4405,7 +4776,22 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, same_page: if (pages) { pages[i] = mem_map_offset(page, pfn_offset); - get_page(pages[i]); + /* + * try_grab_page() should always succeed here, because: + * a) we hold the ptl lock, and b) we've just checked + * that the huge page is present in the page tables. If + * the huge page is present, then the tail pages must + * also be present. The ptl prevents the head page and + * tail pages from being rearranged in any way. So this + * page must be available at this point, unless the page + * refcount overflowed: + */ + if (WARN_ON_ONCE(!try_grab_page(pages[i], flags))) { + spin_unlock(ptl); + remainder = 0; + err = -ENOMEM; + break; + } } if (vmas) @@ -4541,11 +4927,12 @@ int hugetlb_reserve_pages(struct inode *inode, struct vm_area_struct *vma, vm_flags_t vm_flags) { - long ret, chg; + long ret, chg, add = -1; struct hstate *h = hstate_inode(inode); struct hugepage_subpool *spool = subpool_inode(inode); struct resv_map *resv_map; - long gbl_reserve; + struct hugetlb_cgroup *h_cg = NULL; + long gbl_reserve, regions_needed = 0; /* This should never happen */ if (from > to) { @@ -4575,9 +4962,10 @@ int hugetlb_reserve_pages(struct inode *inode, */ resv_map = inode_resv_map(inode); - chg = region_chg(resv_map, from, to); + chg = region_chg(resv_map, from, to, ®ions_needed); } else { + /* Private mapping. */ resv_map = resv_map_alloc(); if (!resv_map) return -ENOMEM; @@ -4593,6 +4981,21 @@ int hugetlb_reserve_pages(struct inode *inode, goto out_err; } + ret = hugetlb_cgroup_charge_cgroup_rsvd( + hstate_index(h), chg * pages_per_huge_page(h), &h_cg); + + if (ret < 0) { + ret = -ENOMEM; + goto out_err; + } + + if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { + /* For private mappings, the hugetlb_cgroup uncharge info hangs + * of the resv_map. + */ + resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h); + } + /* * There must be enough pages in the subpool for the mapping. If * the subpool has a minimum size, there may be some global @@ -4601,7 +5004,7 @@ int hugetlb_reserve_pages(struct inode *inode, gbl_reserve = hugepage_subpool_get_pages(spool, chg); if (gbl_reserve < 0) { ret = -ENOSPC; - goto out_err; + goto out_uncharge_cgroup; } /* @@ -4610,9 +5013,7 @@ int hugetlb_reserve_pages(struct inode *inode, */ ret = hugetlb_acct_memory(h, gbl_reserve); if (ret < 0) { - /* put back original number of pages, chg */ - (void)hugepage_subpool_put_pages(spool, chg); - goto out_err; + goto out_put_pages; } /* @@ -4627,9 +5028,12 @@ int hugetlb_reserve_pages(struct inode *inode, * else has to be done for private mappings here */ if (!vma || vma->vm_flags & VM_MAYSHARE) { - long add = region_add(resv_map, from, to); + add = region_add(resv_map, from, to, regions_needed, h, h_cg); - if (unlikely(chg > add)) { + if (unlikely(add < 0)) { + hugetlb_acct_memory(h, -gbl_reserve); + goto out_put_pages; + } else if (unlikely(chg > add)) { /* * pages in this range were added to the reserve * map between region_chg and region_add. This @@ -4639,17 +5043,29 @@ int hugetlb_reserve_pages(struct inode *inode, */ long rsv_adjust; + hugetlb_cgroup_uncharge_cgroup_rsvd( + hstate_index(h), + (chg - add) * pages_per_huge_page(h), h_cg); + rsv_adjust = hugepage_subpool_put_pages(spool, chg - add); hugetlb_acct_memory(h, -rsv_adjust); } } return 0; +out_put_pages: + /* put back original number of pages, chg */ + (void)hugepage_subpool_put_pages(spool, chg); +out_uncharge_cgroup: + hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), + chg * pages_per_huge_page(h), h_cg); out_err: if (!vma || vma->vm_flags & VM_MAYSHARE) - /* Don't call region_abort if region_chg failed */ - if (chg >= 0) - region_abort(resv_map, from, to); + /* Only call region_abort if the region_chg succeeded but the + * region_add failed or didn't run. + */ + if (chg >= 0 && add < 0) + region_abort(resv_map, from, to, regions_needed); if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) kref_put(&resv_map->refs, resv_map_release); return ret; @@ -4740,7 +5156,7 @@ static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, unsigned long *start, unsigned long *end) { - unsigned long check_addr = *start; + unsigned long check_addr; if (!(vma->vm_flags & VM_MAYSHARE)) return; @@ -4765,10 +5181,12 @@ void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() * and returns the corresponding pte. While this is not necessary for the * !shared pmd case because we can allocate the pmd later as well, it makes the - * code much cleaner. pmd allocation is essential for the shared case because - * pud has to be populated inside the same i_mmap_rwsem section - otherwise - * racing tasks could either miss the sharing (see huge_pte_offset) or select a - * bad pmd for sharing. + * code much cleaner. + * + * This routine must be called with i_mmap_rwsem held in at least read mode. + * For hugetlbfs, this prevents removal of any page table entries associated + * with the address space. This is important as we are setting up sharing + * based on existing page table entries (mappings). */ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) { @@ -4785,7 +5203,6 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) if (!vma_shareable(vma, addr)) return (pte_t *)pmd_alloc(mm, pud, addr); - i_mmap_lock_read(mapping); vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { if (svma == vma) continue; @@ -4815,7 +5232,6 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) spin_unlock(ptl); out: pte = (pte_t *)pmd_alloc(mm, pud, addr); - i_mmap_unlock_read(mapping); return pte; } @@ -4826,7 +5242,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) * indicated by page_count > 1, unmap is achieved by clearing pud and * decrementing the ref count. If count == 1, the pte page is not shared. * - * called with page table lock held. + * Called with page table lock held and i_mmap_rwsem held in write mode. * * returns: 1 successfully unmapped a shared pte page * 0 the underlying pte page is not shared, or it is the last user @@ -4965,6 +5381,12 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, struct page *page = NULL; spinlock_t *ptl; pte_t pte; + + /* FOLL_GET and FOLL_PIN are mutually exclusive. */ + if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == + (FOLL_PIN | FOLL_GET))) + return NULL; + retry: ptl = pmd_lockptr(mm, pmd); spin_lock(ptl); @@ -4977,8 +5399,18 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, pte = huge_ptep_get((pte_t *)pmd); if (pte_present(pte)) { page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT); - if (flags & FOLL_GET) - get_page(page); + /* + * try_grab_page() should always succeed here, because: a) we + * hold the pmd (ptl) lock, and b) we've just checked that the + * huge pmd (head) page is present in the page tables. The ptl + * prevents the head page and tail pages from being rearranged + * in any way. So this page must be available at this point, + * unless the page refcount overflowed: + */ + if (WARN_ON_ONCE(!try_grab_page(page, flags))) { + page = NULL; + goto out; + } } else { if (is_hugetlb_entry_migration(pte)) { spin_unlock(ptl); @@ -4999,7 +5431,7 @@ struct page * __weak follow_huge_pud(struct mm_struct *mm, unsigned long address, pud_t *pud, int flags) { - if (flags & FOLL_GET) + if (flags & (FOLL_GET | FOLL_PIN)) return NULL; return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); @@ -5008,7 +5440,7 @@ follow_huge_pud(struct mm_struct *mm, unsigned long address, struct page * __weak follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags) { - if (flags & FOLL_GET) + if (flags & (FOLL_GET | FOLL_PIN)) return NULL; return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT); diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index 5280bcf459af..c2d7ae6cabd1 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -23,29 +23,6 @@ #include #include -enum hugetlb_memory_event { - HUGETLB_MAX, - HUGETLB_NR_MEMORY_EVENTS, -}; - -struct hugetlb_cgroup { - struct cgroup_subsys_state css; - - /* - * the counter to account for hugepages from hugetlb. - */ - struct page_counter hugepage[HUGE_MAX_HSTATE]; - - atomic_long_t events[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; - atomic_long_t events_local[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; - - /* Handle for "hugetlb.events" */ - struct cgroup_file events_file[HUGE_MAX_HSTATE]; - - /* Handle for "hugetlb.events.local" */ - struct cgroup_file events_local_file[HUGE_MAX_HSTATE]; -}; - #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) #define MEMFILE_IDX(val) (((val) >> 16) & 0xffff) #define MEMFILE_ATTR(val) ((val) & 0xffff) @@ -55,6 +32,27 @@ struct hugetlb_cgroup { static struct hugetlb_cgroup *root_h_cgroup __read_mostly; +static inline struct page_counter * +__hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx, + bool rsvd) +{ + if (rsvd) + return &h_cg->rsvd_hugepage[idx]; + return &h_cg->hugepage[idx]; +} + +static inline struct page_counter * +hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx) +{ + return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false); +} + +static inline struct page_counter * +hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx) +{ + return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true); +} + static inline struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s) { @@ -83,8 +81,12 @@ static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg) int idx; for (idx = 0; idx < hugetlb_max_hstate; idx++) { - if (page_counter_read(&h_cg->hugepage[idx])) + if (page_counter_read( + hugetlb_cgroup_counter_from_cgroup(h_cg, idx)) || + page_counter_read(hugetlb_cgroup_counter_from_cgroup_rsvd( + h_cg, idx))) { return true; + } } return false; } @@ -95,18 +97,34 @@ static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup, int idx; for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) { - struct page_counter *counter = &h_cgroup->hugepage[idx]; - struct page_counter *parent = NULL; + struct page_counter *fault_parent = NULL; + struct page_counter *rsvd_parent = NULL; unsigned long limit; int ret; - if (parent_h_cgroup) - parent = &parent_h_cgroup->hugepage[idx]; - page_counter_init(counter, parent); + if (parent_h_cgroup) { + fault_parent = hugetlb_cgroup_counter_from_cgroup( + parent_h_cgroup, idx); + rsvd_parent = hugetlb_cgroup_counter_from_cgroup_rsvd( + parent_h_cgroup, idx); + } + page_counter_init(hugetlb_cgroup_counter_from_cgroup(h_cgroup, + idx), + fault_parent); + page_counter_init( + hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx), + rsvd_parent); limit = round_down(PAGE_COUNTER_MAX, 1 << huge_page_order(&hstates[idx])); - ret = page_counter_set_max(counter, limit); + + ret = page_counter_set_max( + hugetlb_cgroup_counter_from_cgroup(h_cgroup, idx), + limit); + VM_BUG_ON(ret); + ret = page_counter_set_max( + hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx), + limit); VM_BUG_ON(ret); } } @@ -136,7 +154,6 @@ static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css) kfree(h_cgroup); } - /* * Should be called with hugetlb_lock held. * Since we are holding hugetlb_lock, pages cannot get moved from @@ -213,8 +230,9 @@ static inline void hugetlb_event(struct hugetlb_cgroup *hugetlb, int idx, !hugetlb_cgroup_is_root(hugetlb)); } -int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, - struct hugetlb_cgroup **ptr) +static int __hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, + struct hugetlb_cgroup **ptr, + bool rsvd) { int ret = 0; struct page_counter *counter; @@ -237,50 +255,103 @@ int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, } rcu_read_unlock(); - if (!page_counter_try_charge(&h_cg->hugepage[idx], nr_pages, - &counter)) { + if (!page_counter_try_charge( + __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), + nr_pages, &counter)) { ret = -ENOMEM; hugetlb_event(h_cg, idx, HUGETLB_MAX); + css_put(&h_cg->css); + goto done; } - css_put(&h_cg->css); + /* Reservations take a reference to the css because they do not get + * reparented. + */ + if (!rsvd) + css_put(&h_cg->css); done: *ptr = h_cg; return ret; } +int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, + struct hugetlb_cgroup **ptr) +{ + return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, false); +} + +int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages, + struct hugetlb_cgroup **ptr) +{ + return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, true); +} + /* Should be called with hugetlb_lock held */ -void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, - struct hugetlb_cgroup *h_cg, - struct page *page) +static void __hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg, + struct page *page, bool rsvd) { if (hugetlb_cgroup_disabled() || !h_cg) return; - set_hugetlb_cgroup(page, h_cg); + __set_hugetlb_cgroup(page, h_cg, rsvd); return; } +void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg, + struct page *page) +{ + __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, false); +} + +void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg, + struct page *page) +{ + __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, true); +} + /* * Should be called with hugetlb_lock held */ -void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, - struct page *page) +static void __hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, + struct page *page, bool rsvd) { struct hugetlb_cgroup *h_cg; if (hugetlb_cgroup_disabled()) return; lockdep_assert_held(&hugetlb_lock); - h_cg = hugetlb_cgroup_from_page(page); + h_cg = __hugetlb_cgroup_from_page(page, rsvd); if (unlikely(!h_cg)) return; - set_hugetlb_cgroup(page, NULL); - page_counter_uncharge(&h_cg->hugepage[idx], nr_pages); + __set_hugetlb_cgroup(page, NULL, rsvd); + + page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, + rsvd), + nr_pages); + + if (rsvd) + css_put(&h_cg->css); + return; } -void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, - struct hugetlb_cgroup *h_cg) +void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, + struct page *page) +{ + __hugetlb_cgroup_uncharge_page(idx, nr_pages, page, false); +} + +void hugetlb_cgroup_uncharge_page_rsvd(int idx, unsigned long nr_pages, + struct page *page) +{ + __hugetlb_cgroup_uncharge_page(idx, nr_pages, page, true); +} + +static void __hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg, + bool rsvd) { if (hugetlb_cgroup_disabled() || !h_cg) return; @@ -288,34 +359,91 @@ void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER) return; - page_counter_uncharge(&h_cg->hugepage[idx], nr_pages); - return; + page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, + rsvd), + nr_pages); + + if (rsvd) + css_put(&h_cg->css); +} + +void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg) +{ + __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false); +} + +void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages, + struct hugetlb_cgroup *h_cg) +{ + __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true); +} + +void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, unsigned long start, + unsigned long end) +{ + if (hugetlb_cgroup_disabled() || !resv || !resv->reservation_counter || + !resv->css) + return; + + page_counter_uncharge(resv->reservation_counter, + (end - start) * resv->pages_per_hpage); + css_put(resv->css); +} + +void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, + struct file_region *rg, + unsigned long nr_pages) +{ + if (hugetlb_cgroup_disabled() || !resv || !rg || !nr_pages) + return; + + if (rg->reservation_counter && resv->pages_per_hpage && nr_pages > 0 && + !resv->reservation_counter) { + page_counter_uncharge(rg->reservation_counter, + nr_pages * resv->pages_per_hpage); + css_put(rg->css); + } } enum { RES_USAGE, + RES_RSVD_USAGE, RES_LIMIT, + RES_RSVD_LIMIT, RES_MAX_USAGE, + RES_RSVD_MAX_USAGE, RES_FAILCNT, + RES_RSVD_FAILCNT, }; static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) { struct page_counter *counter; + struct page_counter *rsvd_counter; struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)]; + rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(cft->private)]; switch (MEMFILE_ATTR(cft->private)) { case RES_USAGE: return (u64)page_counter_read(counter) * PAGE_SIZE; + case RES_RSVD_USAGE: + return (u64)page_counter_read(rsvd_counter) * PAGE_SIZE; case RES_LIMIT: return (u64)counter->max * PAGE_SIZE; + case RES_RSVD_LIMIT: + return (u64)rsvd_counter->max * PAGE_SIZE; case RES_MAX_USAGE: return (u64)counter->watermark * PAGE_SIZE; + case RES_RSVD_MAX_USAGE: + return (u64)rsvd_counter->watermark * PAGE_SIZE; case RES_FAILCNT: return counter->failcnt; + case RES_RSVD_FAILCNT: + return rsvd_counter->failcnt; default: BUG(); } @@ -337,10 +465,16 @@ static int hugetlb_cgroup_read_u64_max(struct seq_file *seq, void *v) 1 << huge_page_order(&hstates[idx])); switch (MEMFILE_ATTR(cft->private)) { + case RES_RSVD_USAGE: + counter = &h_cg->rsvd_hugepage[idx]; + /* Fall through. */ case RES_USAGE: val = (u64)page_counter_read(counter); seq_printf(seq, "%llu\n", val * PAGE_SIZE); break; + case RES_RSVD_LIMIT: + counter = &h_cg->rsvd_hugepage[idx]; + /* Fall through. */ case RES_LIMIT: val = (u64)counter->max; if (val == limit) @@ -364,6 +498,7 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, int ret, idx; unsigned long nr_pages; struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); + bool rsvd = false; if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */ return -EINVAL; @@ -377,9 +512,14 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, nr_pages = round_down(nr_pages, 1 << huge_page_order(&hstates[idx])); switch (MEMFILE_ATTR(of_cft(of)->private)) { + case RES_RSVD_LIMIT: + rsvd = true; + /* Fall through. */ case RES_LIMIT: mutex_lock(&hugetlb_limit_mutex); - ret = page_counter_set_max(&h_cg->hugepage[idx], nr_pages); + ret = page_counter_set_max( + __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), + nr_pages); mutex_unlock(&hugetlb_limit_mutex); break; default: @@ -405,18 +545,25 @@ static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { int ret = 0; - struct page_counter *counter; + struct page_counter *counter, *rsvd_counter; struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)]; + rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(of_cft(of)->private)]; switch (MEMFILE_ATTR(of_cft(of)->private)) { case RES_MAX_USAGE: page_counter_reset_watermark(counter); break; + case RES_RSVD_MAX_USAGE: + page_counter_reset_watermark(rsvd_counter); + break; case RES_FAILCNT: counter->failcnt = 0; break; + case RES_RSVD_FAILCNT: + rsvd_counter->failcnt = 0; + break; default: ret = -EINVAL; break; @@ -471,7 +618,7 @@ static void __init __hugetlb_cgroup_file_dfl_init(int idx) struct hstate *h = &hstates[idx]; /* format the size */ - mem_fmt(buf, 32, huge_page_size(h)); + mem_fmt(buf, sizeof(buf), huge_page_size(h)); /* Add the limit file */ cft = &h->cgroup_files_dfl[0]; @@ -481,15 +628,30 @@ static void __init __hugetlb_cgroup_file_dfl_init(int idx) cft->write = hugetlb_cgroup_write_dfl; cft->flags = CFTYPE_NOT_ON_ROOT; - /* Add the current usage file */ + /* Add the reservation limit file */ cft = &h->cgroup_files_dfl[1]; + snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.max", buf); + cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT); + cft->seq_show = hugetlb_cgroup_read_u64_max; + cft->write = hugetlb_cgroup_write_dfl; + cft->flags = CFTYPE_NOT_ON_ROOT; + + /* Add the current usage file */ + cft = &h->cgroup_files_dfl[2]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.current", buf); cft->private = MEMFILE_PRIVATE(idx, RES_USAGE); cft->seq_show = hugetlb_cgroup_read_u64_max; cft->flags = CFTYPE_NOT_ON_ROOT; + /* Add the current reservation usage file */ + cft = &h->cgroup_files_dfl[3]; + snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.current", buf); + cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE); + cft->seq_show = hugetlb_cgroup_read_u64_max; + cft->flags = CFTYPE_NOT_ON_ROOT; + /* Add the events file */ - cft = &h->cgroup_files_dfl[2]; + cft = &h->cgroup_files_dfl[4]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events", buf); cft->private = MEMFILE_PRIVATE(idx, 0); cft->seq_show = hugetlb_events_show; @@ -497,7 +659,7 @@ static void __init __hugetlb_cgroup_file_dfl_init(int idx) cft->flags = CFTYPE_NOT_ON_ROOT; /* Add the events.local file */ - cft = &h->cgroup_files_dfl[3]; + cft = &h->cgroup_files_dfl[5]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events.local", buf); cft->private = MEMFILE_PRIVATE(idx, 0); cft->seq_show = hugetlb_events_local_show; @@ -506,7 +668,7 @@ static void __init __hugetlb_cgroup_file_dfl_init(int idx) cft->flags = CFTYPE_NOT_ON_ROOT; /* NULL terminate the last cft */ - cft = &h->cgroup_files_dfl[4]; + cft = &h->cgroup_files_dfl[6]; memset(cft, 0, sizeof(*cft)); WARN_ON(cgroup_add_dfl_cftypes(&hugetlb_cgrp_subsys, @@ -520,7 +682,7 @@ static void __init __hugetlb_cgroup_file_legacy_init(int idx) struct hstate *h = &hstates[idx]; /* format the size */ - mem_fmt(buf, 32, huge_page_size(h)); + mem_fmt(buf, sizeof(buf), huge_page_size(h)); /* Add the limit file */ cft = &h->cgroup_files_legacy[0]; @@ -529,28 +691,55 @@ static void __init __hugetlb_cgroup_file_legacy_init(int idx) cft->read_u64 = hugetlb_cgroup_read_u64; cft->write = hugetlb_cgroup_write_legacy; - /* Add the usage file */ + /* Add the reservation limit file */ cft = &h->cgroup_files_legacy[1]; + snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.limit_in_bytes", buf); + cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT); + cft->read_u64 = hugetlb_cgroup_read_u64; + cft->write = hugetlb_cgroup_write_legacy; + + /* Add the usage file */ + cft = &h->cgroup_files_legacy[2]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_USAGE); cft->read_u64 = hugetlb_cgroup_read_u64; + /* Add the reservation usage file */ + cft = &h->cgroup_files_legacy[3]; + snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.usage_in_bytes", buf); + cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE); + cft->read_u64 = hugetlb_cgroup_read_u64; + /* Add the MAX usage file */ - cft = &h->cgroup_files_legacy[2]; + cft = &h->cgroup_files_legacy[4]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE); cft->write = hugetlb_cgroup_reset; cft->read_u64 = hugetlb_cgroup_read_u64; + /* Add the MAX reservation usage file */ + cft = &h->cgroup_files_legacy[5]; + snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.max_usage_in_bytes", buf); + cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_MAX_USAGE); + cft->write = hugetlb_cgroup_reset; + cft->read_u64 = hugetlb_cgroup_read_u64; + /* Add the failcntfile */ - cft = &h->cgroup_files_legacy[3]; + cft = &h->cgroup_files_legacy[6]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf); - cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT); + cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT); + cft->write = hugetlb_cgroup_reset; + cft->read_u64 = hugetlb_cgroup_read_u64; + + /* Add the reservation failcntfile */ + cft = &h->cgroup_files_legacy[7]; + snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.failcnt", buf); + cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_FAILCNT); cft->write = hugetlb_cgroup_reset; cft->read_u64 = hugetlb_cgroup_read_u64; /* NULL terminate the last cft */ - cft = &h->cgroup_files_legacy[4]; + cft = &h->cgroup_files_legacy[8]; memset(cft, 0, sizeof(*cft)); WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys, @@ -585,6 +774,7 @@ void __init hugetlb_cgroup_file_init(void) void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage) { struct hugetlb_cgroup *h_cg; + struct hugetlb_cgroup *h_cg_rsvd; struct hstate *h = page_hstate(oldhpage); if (hugetlb_cgroup_disabled()) @@ -593,10 +783,13 @@ void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage) VM_BUG_ON_PAGE(!PageHuge(oldhpage), oldhpage); spin_lock(&hugetlb_lock); h_cg = hugetlb_cgroup_from_page(oldhpage); + h_cg_rsvd = hugetlb_cgroup_from_page_rsvd(oldhpage); set_hugetlb_cgroup(oldhpage, NULL); + set_hugetlb_cgroup_rsvd(oldhpage, NULL); /* move the h_cg details to new cgroup */ set_hugetlb_cgroup(newhpage, h_cg); + set_hugetlb_cgroup_rsvd(newhpage, h_cg_rsvd); list_move(&newhpage->lru, &h->hugepage_activelist); spin_unlock(&hugetlb_lock); return; diff --git a/mm/internal.h b/mm/internal.h index 3cf20ab3ca01..2d58ae15a958 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -63,6 +63,29 @@ static inline unsigned long ra_submit(struct file_ra_state *ra, ra->start, ra->size, ra->async_size); } +/** + * page_evictable - test whether a page is evictable + * @page: the page to test + * + * Test whether page is evictable--i.e., should be placed on active/inactive + * lists vs unevictable list. + * + * Reasons page might not be evictable: + * (1) page's mapping marked unevictable + * (2) page is part of an mlocked VMA + * + */ +static inline bool page_evictable(struct page *page) +{ + bool ret; + + /* Prevent address_space of inode and swap cache from being freed */ + rcu_read_lock(); + ret = !mapping_unevictable(page_mapping(page)) && !PageMlocked(page); + rcu_read_unlock(); + return ret; +} + /* * Turn a non-refcounted page (->_refcount == 0) into refcounted with * a count of one. @@ -206,6 +229,7 @@ struct compact_control { bool whole_zone; /* Whole zone should/has been scanned */ bool contended; /* Signal lock or sched contention */ bool rescan; /* Rescanning the same pageblock */ + bool alloc_contig; /* alloc_contig_range allocation */ }; /* @@ -377,10 +401,10 @@ static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, /* * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or * anything, so we only pin the file and drop the mmap_sem if only - * FAULT_FLAG_ALLOW_RETRY is set. + * FAULT_FLAG_ALLOW_RETRY is set, while this is the first attempt. */ - if ((flags & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT)) == - FAULT_FLAG_ALLOW_RETRY) { + if (fault_flag_allow_retry_first(flags) && + !(flags & FAULT_FLAG_RETRY_NOWAIT)) { fpin = get_file(vmf->vma->vm_file); up_read(&vmf->vma->vm_mm->mmap_sem); } @@ -532,7 +556,7 @@ unsigned long reclaim_clean_pages_from_list(struct zone *zone, #else #define ALLOC_NOFRAGMENT 0x0 #endif -#define ALLOC_KSWAPD 0x200 /* allow waking of kswapd */ +#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ enum ttu_flags; struct tlbflush_unmap_batch; diff --git a/mm/kasan/common.c b/mm/kasan/common.c index 6aa51723b92b..e61b4a492218 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -105,7 +105,8 @@ EXPORT_SYMBOL(__kasan_check_write); #undef memset void *memset(void *addr, int c, size_t len) { - check_memory_region((unsigned long)addr, len, true, _RET_IP_); + if (!check_memory_region((unsigned long)addr, len, true, _RET_IP_)) + return NULL; return __memset(addr, c, len); } @@ -114,8 +115,9 @@ void *memset(void *addr, int c, size_t len) #undef memmove void *memmove(void *dest, const void *src, size_t len) { - check_memory_region((unsigned long)src, len, false, _RET_IP_); - check_memory_region((unsigned long)dest, len, true, _RET_IP_); + if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) || + !check_memory_region((unsigned long)dest, len, true, _RET_IP_)) + return NULL; return __memmove(dest, src, len); } @@ -124,8 +126,9 @@ void *memmove(void *dest, const void *src, size_t len) #undef memcpy void *memcpy(void *dest, const void *src, size_t len) { - check_memory_region((unsigned long)src, len, false, _RET_IP_); - check_memory_region((unsigned long)dest, len, true, _RET_IP_); + if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) || + !check_memory_region((unsigned long)dest, len, true, _RET_IP_)) + return NULL; return __memcpy(dest, src, len); } @@ -634,12 +637,21 @@ void kasan_free_shadow(const struct vm_struct *vm) #endif extern void __kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip); +extern bool report_enabled(void); -void kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip) +bool kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip) { unsigned long flags = user_access_save(); - __kasan_report(addr, size, is_write, ip); + bool ret = false; + + if (likely(report_enabled())) { + __kasan_report(addr, size, is_write, ip); + ret = true; + } + user_access_restore(flags); + + return ret; } #ifdef CONFIG_MEMORY_HOTPLUG diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 616f9dd82d12..56ff8885fe2e 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -173,17 +173,18 @@ static __always_inline bool check_memory_region_inline(unsigned long addr, if (unlikely(size == 0)) return true; + if (unlikely(addr + size < addr)) + return !kasan_report(addr, size, write, ret_ip); + if (unlikely((void *)addr < kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) { - kasan_report(addr, size, write, ret_ip); - return false; + return !kasan_report(addr, size, write, ret_ip); } if (likely(!memory_is_poisoned(addr, size))) return true; - kasan_report(addr, size, write, ret_ip); - return false; + return !kasan_report(addr, size, write, ret_ip); } bool check_memory_region(unsigned long addr, size_t size, bool write, diff --git a/mm/kasan/generic_report.c b/mm/kasan/generic_report.c index 2d97efd4954f..e200acb2d292 100644 --- a/mm/kasan/generic_report.c +++ b/mm/kasan/generic_report.c @@ -110,6 +110,17 @@ static const char *get_wild_bug_type(struct kasan_access_info *info) const char *get_bug_type(struct kasan_access_info *info) { + /* + * If access_size is a negative number, then it has reason to be + * defined as out-of-bounds bug type. + * + * Casting negative numbers to size_t would indeed turn up as + * a large size_t and its value will be larger than ULONG_MAX/2, + * so that this can qualify as out-of-bounds. + */ + if (info->access_addr + info->access_size < info->access_addr) + return "out-of-bounds"; + if (addr_has_shadow(info->access_addr)) return get_shadow_bug_type(info); return get_wild_bug_type(info); diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 3a083274628e..e8f37199d885 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -153,7 +153,7 @@ bool check_memory_region(unsigned long addr, size_t size, bool write, void *find_first_bad_addr(void *addr, size_t size); const char *get_bug_type(struct kasan_access_info *info); -void kasan_report(unsigned long addr, size_t size, +bool kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip); void kasan_report_invalid_free(void *object, unsigned long ip); diff --git a/mm/kasan/report.c b/mm/kasan/report.c index 5ef9f24f566b..cf5c17d5e361 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -446,7 +446,7 @@ static void print_shadow_for_address(const void *addr) } } -static bool report_enabled(void) +bool report_enabled(void) { if (current->kasan_depth) return false; @@ -478,9 +478,6 @@ void __kasan_report(unsigned long addr, size_t size, bool is_write, unsigned lon void *untagged_addr; unsigned long flags; - if (likely(!report_enabled())) - return; - disable_trace_on_warning(); tagged_addr = (void *)addr; diff --git a/mm/kasan/tags.c b/mm/kasan/tags.c index 0e987c9ca052..25b7734e7013 100644 --- a/mm/kasan/tags.c +++ b/mm/kasan/tags.c @@ -86,6 +86,9 @@ bool check_memory_region(unsigned long addr, size_t size, bool write, if (unlikely(size == 0)) return true; + if (unlikely(addr + size < addr)) + return !kasan_report(addr, size, write, ret_ip); + tag = get_tag((const void *)addr); /* @@ -111,15 +114,13 @@ bool check_memory_region(unsigned long addr, size_t size, bool write, untagged_addr = reset_tag((const void *)addr); if (unlikely(untagged_addr < kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) { - kasan_report(addr, size, write, ret_ip); - return false; + return !kasan_report(addr, size, write, ret_ip); } shadow_first = kasan_mem_to_shadow(untagged_addr); shadow_last = kasan_mem_to_shadow(untagged_addr + size - 1); for (shadow = shadow_first; shadow <= shadow_last; shadow++) { if (*shadow != tag) { - kasan_report(addr, size, write, ret_ip); - return false; + return !kasan_report(addr, size, write, ret_ip); } } diff --git a/mm/kasan/tags_report.c b/mm/kasan/tags_report.c index 969ae08f59d7..bee43717d6f0 100644 --- a/mm/kasan/tags_report.c +++ b/mm/kasan/tags_report.c @@ -60,6 +60,17 @@ const char *get_bug_type(struct kasan_access_info *info) } #endif + /* + * If access_size is a negative number, then it has reason to be + * defined as out-of-bounds bug type. + * + * Casting negative numbers to size_t would indeed turn up as + * a large size_t and its value will be larger than ULONG_MAX/2, + * so that this can qualify as out-of-bounds. + */ + if (info->access_addr + info->access_size < info->access_addr) + return "out-of-bounds"; + return "invalid-access"; } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index b679908743cb..c659c68728bc 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -308,8 +308,6 @@ struct attribute_group khugepaged_attr_group = { }; #endif /* CONFIG_SYSFS */ -#define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB) - int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, int advice) { @@ -423,7 +421,7 @@ static bool hugepage_vma_check(struct vm_area_struct *vma, } if (!vma->anon_vma || vma->vm_ops) return false; - if (is_vma_temporary_stack(vma)) + if (vma_is_temporary_stack(vma)) return false; return !(vm_flags & VM_NO_KHUGEPAGED); } diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 3a4259eeb5a0..e362dc3d2028 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1947,7 +1947,7 @@ void __init kmemleak_init(void) create_object((unsigned long)__bss_start, __bss_stop - __bss_start, KMEMLEAK_GREY, GFP_ATOMIC); /* only register .data..ro_after_init if not within .data */ - if (__start_ro_after_init < _sdata || __end_ro_after_init > _edata) + if (&__start_ro_after_init < &_sdata || &__end_ro_after_init > &_edata) create_object((unsigned long)__start_ro_after_init, __end_ro_after_init - __start_ro_after_init, KMEMLEAK_GREY, GFP_ATOMIC); diff --git a/mm/list_lru.c b/mm/list_lru.c index 0f1f6b06b7f3..8de5e3784ee4 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -57,16 +57,6 @@ list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx) return &nlru->lru; } -static __always_inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr) -{ - struct page *page; - - if (!memcg_kmem_enabled()) - return NULL; - page = virt_to_head_page(ptr); - return memcg_from_slab_page(page); -} - static inline struct list_lru_one * list_lru_from_kmem(struct list_lru_node *nlru, void *ptr, struct mem_cgroup **memcg_ptr) @@ -77,7 +67,7 @@ list_lru_from_kmem(struct list_lru_node *nlru, void *ptr, if (!nlru->memcg_lrus) goto out; - memcg = mem_cgroup_from_kmem(ptr); + memcg = mem_cgroup_from_obj(ptr); if (!memcg) goto out; diff --git a/mm/mapping_dirty_helpers.c b/mm/mapping_dirty_helpers.c index 71070dda9643..2c7d03675903 100644 --- a/mm/mapping_dirty_helpers.c +++ b/mm/mapping_dirty_helpers.c @@ -111,26 +111,60 @@ static int clean_record_pte(pte_t *pte, unsigned long addr, return 0; } -/* wp_clean_pmd_entry - The pagewalk pmd callback. */ +/* + * wp_clean_pmd_entry - The pagewalk pmd callback. + * + * Dirty-tracking should take place on the PTE level, so + * WARN() if encountering a dirty huge pmd. + * Furthermore, never split huge pmds, since that currently + * causes dirty info loss. The pagefault handler should do + * that if needed. + */ static int wp_clean_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) { - /* Dirty-tracking should be handled on the pte level */ pmd_t pmdval = pmd_read_atomic(pmd); + if (!pmd_trans_unstable(&pmdval)) + return 0; + + if (pmd_none(pmdval)) { + walk->action = ACTION_AGAIN; + return 0; + } + + /* Huge pmd, present or migrated */ + walk->action = ACTION_CONTINUE; if (pmd_trans_huge(pmdval) || pmd_devmap(pmdval)) WARN_ON(pmd_write(pmdval) || pmd_dirty(pmdval)); return 0; } -/* wp_clean_pud_entry - The pagewalk pud callback. */ +/* + * wp_clean_pud_entry - The pagewalk pud callback. + * + * Dirty-tracking should take place on the PTE level, so + * WARN() if encountering a dirty huge puds. + * Furthermore, never split huge puds, since that currently + * causes dirty info loss. The pagefault handler should do + * that if needed. + */ static int wp_clean_pud_entry(pud_t *pud, unsigned long addr, unsigned long end, struct mm_walk *walk) { - /* Dirty-tracking should be handled on the pte level */ pud_t pudval = READ_ONCE(*pud); + if (!pud_trans_unstable(&pudval)) + return 0; + + if (pud_none(pudval)) { + walk->action = ACTION_AGAIN; + return 0; + } + + /* Huge pud */ + walk->action = ACTION_CONTINUE; if (pud_trans_huge(pudval) || pud_devmap(pudval)) WARN_ON(pud_write(pudval) || pud_dirty(pudval)); diff --git a/mm/memblock.c b/mm/memblock.c index eba94ee3de0b..4d06bbaded0f 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1698,7 +1698,7 @@ static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit) void __init memblock_enforce_memory_limit(phys_addr_t limit) { - phys_addr_t max_addr = PHYS_ADDR_MAX; + phys_addr_t max_addr; if (!limit) return; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 7ddf91c4295f..ca194864d802 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -334,7 +334,7 @@ static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg, if (!old) return 0; - new = kvmalloc(sizeof(*new) + size, GFP_KERNEL); + new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid); if (!new) return -ENOMEM; @@ -378,7 +378,7 @@ static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg) mutex_lock(&memcg_shrinker_map_mutex); size = memcg_shrinker_map_size; for_each_node(nid) { - map = kvzalloc(sizeof(*map) + size, GFP_KERNEL); + map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid); if (!map) { memcg_free_shrinker_maps(memcg); ret = -ENOMEM; @@ -656,7 +656,7 @@ __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) */ __mem_cgroup_remove_exceeded(mz, mctz); if (!soft_limit_excess(mz->memcg) || - !css_tryget_online(&mz->memcg->css)) + !css_tryget(&mz->memcg->css)) goto retry; done: return mz; @@ -759,13 +759,12 @@ void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val) { - struct page *page = virt_to_head_page(p); - pg_data_t *pgdat = page_pgdat(page); + pg_data_t *pgdat = page_pgdat(virt_to_page(p)); struct mem_cgroup *memcg; struct lruvec *lruvec; rcu_read_lock(); - memcg = memcg_from_slab_page(page); + memcg = mem_cgroup_from_obj(p); /* Untracked pages have no memcg, no lruvec. Update only the node */ if (!memcg || memcg == root_mem_cgroup) { @@ -973,7 +972,8 @@ struct mem_cgroup *get_mem_cgroup_from_page(struct page *page) return NULL; rcu_read_lock(); - if (!memcg || !css_tryget_online(&memcg->css)) + /* Page should not get uncharged and freed memcg under us. */ + if (!memcg || WARN_ON_ONCE(!css_tryget(&memcg->css))) memcg = root_mem_cgroup; rcu_read_unlock(); return memcg; @@ -986,10 +986,13 @@ EXPORT_SYMBOL(get_mem_cgroup_from_page); static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void) { if (unlikely(current->active_memcg)) { - struct mem_cgroup *memcg = root_mem_cgroup; + struct mem_cgroup *memcg; rcu_read_lock(); - if (css_tryget_online(¤t->active_memcg->css)) + /* current->active_memcg must hold a ref. */ + if (WARN_ON_ONCE(!css_tryget(¤t->active_memcg->css))) + memcg = root_mem_cgroup; + else memcg = current->active_memcg; rcu_read_unlock(); return memcg; @@ -1518,11 +1521,11 @@ void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n", K((u64)page_counter_read(&memcg->memory)), - K((u64)memcg->memory.max), memcg->memory.failcnt); + K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt); if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n", K((u64)page_counter_read(&memcg->swap)), - K((u64)memcg->swap.max), memcg->swap.failcnt); + K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt); else { pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n", K((u64)page_counter_read(&memcg->memsw)), @@ -1549,13 +1552,13 @@ unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) { unsigned long max; - max = memcg->memory.max; + max = READ_ONCE(memcg->memory.max); if (mem_cgroup_swappiness(memcg)) { unsigned long memsw_max; unsigned long swap_max; memsw_max = memcg->memsw.max; - swap_max = memcg->swap.max; + swap_max = READ_ONCE(memcg->swap.max); swap_max = min(swap_max, (unsigned long)total_swap_pages); max = min(max + swap_max, memsw_max); } @@ -1927,6 +1930,14 @@ struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim, if (memcg == root_mem_cgroup) goto out; + /* + * If the victim task has been asynchronously moved to a different + * memory cgroup, we might end up killing tasks outside oom_domain. + * In this case it's better to ignore memory.group.oom. + */ + if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain))) + goto out; + /* * Traverse the memory cgroup hierarchy from the victim task's * cgroup up to the OOMing cgroup (or root) to find the @@ -2239,7 +2250,7 @@ static void reclaim_high(struct mem_cgroup *memcg, gfp_t gfp_mask) { do { - if (page_counter_read(&memcg->memory) <= memcg->high) + if (page_counter_read(&memcg->memory) <= READ_ONCE(memcg->high)) continue; memcg_memory_event(memcg, MEMCG_HIGH); try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true); @@ -2579,7 +2590,7 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, * reclaim, the cost of mismatch is negligible. */ do { - if (page_counter_read(&memcg->memory) > memcg->high) { + if (page_counter_read(&memcg->memory) > READ_ONCE(memcg->high)) { /* Don't bother a random interrupted task */ if (in_interrupt()) { schedule_work(&memcg->high_work); @@ -2882,18 +2893,16 @@ void memcg_kmem_put_cache(struct kmem_cache *cachep) } /** - * __memcg_kmem_charge_memcg: charge a kmem page - * @page: page to charge - * @gfp: reclaim mode - * @order: allocation order + * __memcg_kmem_charge: charge a number of kernel pages to a memcg * @memcg: memory cgroup to charge + * @gfp: reclaim mode + * @nr_pages: number of pages to charge * * Returns 0 on success, an error code on failure. */ -int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, - struct mem_cgroup *memcg) +int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp, + unsigned int nr_pages) { - unsigned int nr_pages = 1 << order; struct page_counter *counter; int ret; @@ -2920,14 +2929,29 @@ int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, } /** - * __memcg_kmem_charge: charge a kmem page to the current memory cgroup + * __memcg_kmem_uncharge: uncharge a number of kernel pages from a memcg + * @memcg: memcg to uncharge + * @nr_pages: number of pages to uncharge + */ +void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages) +{ + if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) + page_counter_uncharge(&memcg->kmem, nr_pages); + + page_counter_uncharge(&memcg->memory, nr_pages); + if (do_memsw_account()) + page_counter_uncharge(&memcg->memsw, nr_pages); +} + +/** + * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup * @page: page to charge * @gfp: reclaim mode * @order: allocation order * * Returns 0 on success, an error code on failure. */ -int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order) +int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order) { struct mem_cgroup *memcg; int ret = 0; @@ -2937,7 +2961,7 @@ int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order) memcg = get_mem_cgroup_from_current(); if (!mem_cgroup_is_root(memcg)) { - ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg); + ret = __memcg_kmem_charge(memcg, gfp, 1 << order); if (!ret) { page->mem_cgroup = memcg; __SetPageKmemcg(page); @@ -2948,26 +2972,11 @@ int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order) } /** - * __memcg_kmem_uncharge_memcg: uncharge a kmem page - * @memcg: memcg to uncharge - * @nr_pages: number of pages to uncharge - */ -void __memcg_kmem_uncharge_memcg(struct mem_cgroup *memcg, - unsigned int nr_pages) -{ - if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) - page_counter_uncharge(&memcg->kmem, nr_pages); - - page_counter_uncharge(&memcg->memory, nr_pages); - if (do_memsw_account()) - page_counter_uncharge(&memcg->memsw, nr_pages); -} -/** - * __memcg_kmem_uncharge: uncharge a kmem page + * __memcg_kmem_uncharge_page: uncharge a kmem page * @page: page to uncharge * @order: allocation order */ -void __memcg_kmem_uncharge(struct page *page, int order) +void __memcg_kmem_uncharge_page(struct page *page, int order) { struct mem_cgroup *memcg = page->mem_cgroup; unsigned int nr_pages = 1 << order; @@ -2976,7 +2985,7 @@ void __memcg_kmem_uncharge(struct page *page, int order) return; VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page); - __memcg_kmem_uncharge_memcg(memcg, nr_pages); + __memcg_kmem_uncharge(memcg, nr_pages); page->mem_cgroup = NULL; /* slab pages do not have PageKmemcg flag set */ @@ -3067,7 +3076,7 @@ static int mem_cgroup_resize_max(struct mem_cgroup *memcg, * Make sure that the new limit (memsw or memory limit) doesn't * break our basic invariant rule memory.max <= memsw.max. */ - limits_invariant = memsw ? max >= memcg->memory.max : + limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) : max <= memcg->memsw.max; if (!limits_invariant) { mutex_unlock(&memcg_max_mutex); @@ -3814,8 +3823,8 @@ static int memcg_stat_show(struct seq_file *m, void *v) /* Hierarchical information */ memory = memsw = PAGE_COUNTER_MAX; for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) { - memory = min(memory, mi->memory.max); - memsw = min(memsw, mi->memsw.max); + memory = min(memory, READ_ONCE(mi->memory.max)); + memsw = min(memsw, READ_ONCE(mi->memsw.max)); } seq_printf(m, "hierarchical_memory_limit %llu\n", (u64)memory * PAGE_SIZE); @@ -4324,7 +4333,8 @@ void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, *pheadroom = PAGE_COUNTER_MAX; while ((parent = parent_mem_cgroup(memcg))) { - unsigned long ceiling = min(memcg->memory.max, memcg->high); + unsigned long ceiling = min(READ_ONCE(memcg->memory.max), + READ_ONCE(memcg->high)); unsigned long used = page_counter_read(&memcg->memory); *pheadroom = min(*pheadroom, ceiling - min(ceiling, used)); @@ -4792,7 +4802,8 @@ static struct cftype mem_cgroup_legacy_files[] = { .write = mem_cgroup_reset, .read_u64 = mem_cgroup_read_u64, }, -#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG) +#if defined(CONFIG_MEMCG_KMEM) && \ + (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)) { .name = "kmem.slabinfo", .seq_start = memcg_slab_start, @@ -4861,7 +4872,8 @@ static void mem_cgroup_id_remove(struct mem_cgroup *memcg) } } -static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n) +static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg, + unsigned int n) { refcount_add(n, &memcg->id.ref); } @@ -5044,7 +5056,7 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) if (!memcg) return ERR_PTR(error); - memcg->high = PAGE_COUNTER_MAX; + WRITE_ONCE(memcg->high, PAGE_COUNTER_MAX); memcg->soft_limit = PAGE_COUNTER_MAX; if (parent) { memcg->swappiness = mem_cgroup_swappiness(parent); @@ -5197,7 +5209,7 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); page_counter_set_min(&memcg->memory, 0); page_counter_set_low(&memcg->memory, 0); - memcg->high = PAGE_COUNTER_MAX; + WRITE_ONCE(memcg->high, PAGE_COUNTER_MAX); memcg->soft_limit = PAGE_COUNTER_MAX; memcg_wb_domain_size_changed(memcg); } @@ -6013,7 +6025,7 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, if (err) return err; - memcg->high = high; + WRITE_ONCE(memcg->high, high); for (;;) { unsigned long nr_pages = page_counter_read(&memcg->memory); @@ -6236,6 +6248,117 @@ struct cgroup_subsys memory_cgrp_subsys = { .early_init = 0, }; +/* + * This function calculates an individual cgroup's effective + * protection which is derived from its own memory.min/low, its + * parent's and siblings' settings, as well as the actual memory + * distribution in the tree. + * + * The following rules apply to the effective protection values: + * + * 1. At the first level of reclaim, effective protection is equal to + * the declared protection in memory.min and memory.low. + * + * 2. To enable safe delegation of the protection configuration, at + * subsequent levels the effective protection is capped to the + * parent's effective protection. + * + * 3. To make complex and dynamic subtrees easier to configure, the + * user is allowed to overcommit the declared protection at a given + * level. If that is the case, the parent's effective protection is + * distributed to the children in proportion to how much protection + * they have declared and how much of it they are utilizing. + * + * This makes distribution proportional, but also work-conserving: + * if one cgroup claims much more protection than it uses memory, + * the unused remainder is available to its siblings. + * + * 4. Conversely, when the declared protection is undercommitted at a + * given level, the distribution of the larger parental protection + * budget is NOT proportional. A cgroup's protection from a sibling + * is capped to its own memory.min/low setting. + * + * 5. However, to allow protecting recursive subtrees from each other + * without having to declare each individual cgroup's fixed share + * of the ancestor's claim to protection, any unutilized - + * "floating" - protection from up the tree is distributed in + * proportion to each cgroup's *usage*. This makes the protection + * neutral wrt sibling cgroups and lets them compete freely over + * the shared parental protection budget, but it protects the + * subtree as a whole from neighboring subtrees. + * + * Note that 4. and 5. are not in conflict: 4. is about protecting + * against immediate siblings whereas 5. is about protecting against + * neighboring subtrees. + */ +static unsigned long effective_protection(unsigned long usage, + unsigned long parent_usage, + unsigned long setting, + unsigned long parent_effective, + unsigned long siblings_protected) +{ + unsigned long protected; + unsigned long ep; + + protected = min(usage, setting); + /* + * If all cgroups at this level combined claim and use more + * protection then what the parent affords them, distribute + * shares in proportion to utilization. + * + * We are using actual utilization rather than the statically + * claimed protection in order to be work-conserving: claimed + * but unused protection is available to siblings that would + * otherwise get a smaller chunk than what they claimed. + */ + if (siblings_protected > parent_effective) + return protected * parent_effective / siblings_protected; + + /* + * Ok, utilized protection of all children is within what the + * parent affords them, so we know whatever this child claims + * and utilizes is effectively protected. + * + * If there is unprotected usage beyond this value, reclaim + * will apply pressure in proportion to that amount. + * + * If there is unutilized protection, the cgroup will be fully + * shielded from reclaim, but we do return a smaller value for + * protection than what the group could enjoy in theory. This + * is okay. With the overcommit distribution above, effective + * protection is always dependent on how memory is actually + * consumed among the siblings anyway. + */ + ep = protected; + + /* + * If the children aren't claiming (all of) the protection + * afforded to them by the parent, distribute the remainder in + * proportion to the (unprotected) memory of each cgroup. That + * way, cgroups that aren't explicitly prioritized wrt each + * other compete freely over the allowance, but they are + * collectively protected from neighboring trees. + * + * We're using unprotected memory for the weight so that if + * some cgroups DO claim explicit protection, we don't protect + * the same bytes twice. + */ + if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT)) + return ep; + + if (parent_effective > siblings_protected && usage > protected) { + unsigned long unclaimed; + + unclaimed = parent_effective - siblings_protected; + unclaimed *= usage - protected; + unclaimed /= parent_usage - siblings_protected; + + ep += unclaimed; + } + + return ep; +} + /** * mem_cgroup_protected - check if memory consumption is in the normal range * @root: the top ancestor of the sub-tree being checked @@ -6249,70 +6372,12 @@ struct cgroup_subsys memory_cgrp_subsys = { * MEMCG_PROT_LOW: cgroup memory is protected as long there is * an unprotected supply of reclaimable memory from other cgroups. * MEMCG_PROT_MIN: cgroup memory is protected - * - * @root is exclusive; it is never protected when looked at directly - * - * To provide a proper hierarchical behavior, effective memory.min/low values - * are used. Below is the description of how effective memory.low is calculated. - * Effective memory.min values is calculated in the same way. - * - * Effective memory.low is always equal or less than the original memory.low. - * If there is no memory.low overcommittment (which is always true for - * top-level memory cgroups), these two values are equal. - * Otherwise, it's a part of parent's effective memory.low, - * calculated as a cgroup's memory.low usage divided by sum of sibling's - * memory.low usages, where memory.low usage is the size of actually - * protected memory. - * - * low_usage - * elow = min( memory.low, parent->elow * ------------------ ), - * siblings_low_usage - * - * | memory.current, if memory.current < memory.low - * low_usage = | - * | 0, otherwise. - * - * - * Such definition of the effective memory.low provides the expected - * hierarchical behavior: parent's memory.low value is limiting - * children, unprotected memory is reclaimed first and cgroups, - * which are not using their guarantee do not affect actual memory - * distribution. - * - * For example, if there are memcgs A, A/B, A/C, A/D and A/E: - * - * A A/memory.low = 2G, A/memory.current = 6G - * //\\ - * BC DE B/memory.low = 3G B/memory.current = 2G - * C/memory.low = 1G C/memory.current = 2G - * D/memory.low = 0 D/memory.current = 2G - * E/memory.low = 10G E/memory.current = 0 - * - * and the memory pressure is applied, the following memory distribution - * is expected (approximately): - * - * A/memory.current = 2G - * - * B/memory.current = 1.3G - * C/memory.current = 0.6G - * D/memory.current = 0 - * E/memory.current = 0 - * - * These calculations require constant tracking of the actual low usages - * (see propagate_protected_usage()), as well as recursive calculation of - * effective memory.low values. But as we do call mem_cgroup_protected() - * path for each memory cgroup top-down from the reclaim, - * it's possible to optimize this part, and save calculated elow - * for next usage. This part is intentionally racy, but it's ok, - * as memory.low is a best-effort mechanism. */ enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root, struct mem_cgroup *memcg) { + unsigned long usage, parent_usage; struct mem_cgroup *parent; - unsigned long emin, parent_emin; - unsigned long elow, parent_elow; - unsigned long usage; if (mem_cgroup_disabled()) return MEMCG_PROT_NONE; @@ -6326,52 +6391,32 @@ enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root, if (!usage) return MEMCG_PROT_NONE; - emin = memcg->memory.min; - elow = memcg->memory.low; - parent = parent_mem_cgroup(memcg); /* No parent means a non-hierarchical mode on v1 memcg */ if (!parent) return MEMCG_PROT_NONE; - if (parent == root) - goto exit; - - parent_emin = READ_ONCE(parent->memory.emin); - emin = min(emin, parent_emin); - if (emin && parent_emin) { - unsigned long min_usage, siblings_min_usage; - - min_usage = min(usage, memcg->memory.min); - siblings_min_usage = atomic_long_read( - &parent->memory.children_min_usage); - - if (min_usage && siblings_min_usage) - emin = min(emin, parent_emin * min_usage / - siblings_min_usage); + if (parent == root) { + memcg->memory.emin = READ_ONCE(memcg->memory.min); + memcg->memory.elow = memcg->memory.low; + goto out; } - parent_elow = READ_ONCE(parent->memory.elow); - elow = min(elow, parent_elow); - if (elow && parent_elow) { - unsigned long low_usage, siblings_low_usage; + parent_usage = page_counter_read(&parent->memory); - low_usage = min(usage, memcg->memory.low); - siblings_low_usage = atomic_long_read( - &parent->memory.children_low_usage); + WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage, + READ_ONCE(memcg->memory.min), + READ_ONCE(parent->memory.emin), + atomic_long_read(&parent->memory.children_min_usage))); - if (low_usage && siblings_low_usage) - elow = min(elow, parent_elow * low_usage / - siblings_low_usage); - } + WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage, + memcg->memory.low, READ_ONCE(parent->memory.elow), + atomic_long_read(&parent->memory.children_low_usage))); -exit: - memcg->memory.emin = emin; - memcg->memory.elow = elow; - - if (usage <= emin) +out: + if (usage <= memcg->memory.emin) return MEMCG_PROT_MIN; - else if (usage <= elow) + else if (usage <= memcg->memory.elow) return MEMCG_PROT_LOW; else return MEMCG_PROT_NONE; @@ -6759,7 +6804,7 @@ void mem_cgroup_sk_alloc(struct sock *sk) goto out; if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active) goto out; - if (css_tryget_online(&memcg->css)) + if (css_tryget(&memcg->css)) sk->sk_memcg = memcg; out: rcu_read_unlock(); @@ -7080,7 +7125,8 @@ bool mem_cgroup_swap_full(struct page *page) return false; for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) - if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.max) + if (page_counter_read(&memcg->swap) * 2 >= + READ_ONCE(memcg->swap.max)) return true; return false; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 41c634f45d45..1c961cd26c0b 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -954,7 +954,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS; struct address_space *mapping; LIST_HEAD(tokill); - bool unmap_success; + bool unmap_success = true; int kill = 1, forcekill; struct page *hpage = *hpagep; bool mlocked = PageMlocked(hpage); @@ -1016,7 +1016,32 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, if (kill) collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED); - unmap_success = try_to_unmap(hpage, ttu); + if (!PageHuge(hpage)) { + unmap_success = try_to_unmap(hpage, ttu); + } else { + /* + * For hugetlb pages, try_to_unmap could potentially call + * huge_pmd_unshare. Because of this, take semaphore in + * write mode here and set TTU_RMAP_LOCKED to indicate we + * have taken the lock at this higer level. + * + * Note that the call to hugetlb_page_mapping_lock_write + * is necessary even if mapping is already set. It handles + * ugliness of potentially having to drop page lock to obtain + * i_mmap_rwsem. + */ + mapping = hugetlb_page_mapping_lock_write(hpage); + + if (mapping) { + unmap_success = try_to_unmap(hpage, + ttu|TTU_RMAP_LOCKED); + i_mmap_unlock_write(mapping); + } else { + pr_info("Memory failure: %#lx: could not find mapping for mapped huge page\n", + pfn); + unmap_success = false; + } + } if (!unmap_success) pr_err("Memory failure: %#lx: failed to unmap page (mapcount=%d)\n", pfn, page_mapcount(hpage)); diff --git a/mm/memory.c b/mm/memory.c index 5ddf44efc46e..85de5890e369 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1952,7 +1952,7 @@ static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd, * remap_pfn_range - remap kernel memory to userspace * @vma: user vma to map to * @addr: target user address to start at - * @pfn: physical address of kernel memory + * @pfn: page frame number of kernel physical memory address * @size: size of map area * @prot: page protection flags for this mapping * @@ -2022,7 +2022,7 @@ EXPORT_SYMBOL(remap_pfn_range); /** * vm_iomap_memory - remap memory to userspace * @vma: user vma to map to - * @start: start of area + * @start: start of the physical memory to be mapped * @len: size of area * * This is a simplified io_remap_pfn_range() for common driver use. The diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 59c8ffa1b6b5..13b3d0bb7bb2 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -557,9 +557,10 @@ static int queue_pages_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long end, struct mm_walk *walk) { + int ret = 0; #ifdef CONFIG_HUGETLB_PAGE struct queue_pages *qp = walk->private; - unsigned long flags = qp->flags; + unsigned long flags = (qp->flags & MPOL_MF_VALID); struct page *page; spinlock_t *ptl; pte_t entry; @@ -571,16 +572,44 @@ static int queue_pages_hugetlb(pte_t *pte, unsigned long hmask, page = pte_page(entry); if (!queue_pages_required(page, qp)) goto unlock; + + if (flags == MPOL_MF_STRICT) { + /* + * STRICT alone means only detecting misplaced page and no + * need to further check other vma. + */ + ret = -EIO; + goto unlock; + } + + if (!vma_migratable(walk->vma)) { + /* + * Must be STRICT with MOVE*, otherwise .test_walk() have + * stopped walking current vma. + * Detecting misplaced page but allow migrating pages which + * have been queued. + */ + ret = 1; + goto unlock; + } + /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */ if (flags & (MPOL_MF_MOVE_ALL) || - (flags & MPOL_MF_MOVE && page_mapcount(page) == 1)) - isolate_huge_page(page, qp->pagelist); + (flags & MPOL_MF_MOVE && page_mapcount(page) == 1)) { + if (!isolate_huge_page(page, qp->pagelist) && + (flags & MPOL_MF_STRICT)) + /* + * Failed to isolate page but allow migrating pages + * which have been queued. + */ + ret = 1; + } unlock: spin_unlock(ptl); #else BUG(); #endif - return 0; + return ret; } #ifdef CONFIG_NUMA_BALANCING @@ -621,7 +650,7 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, unsigned long flags = qp->flags; /* range check first */ - VM_BUG_ON((vma->vm_start > start) || (vma->vm_end < end)); + VM_BUG_ON_VMA((vma->vm_start > start) || (vma->vm_end < end), vma); if (!qp->first) { qp->first = vma; @@ -1715,6 +1744,34 @@ COMPAT_SYSCALL_DEFINE4(migrate_pages, compat_pid_t, pid, #endif /* CONFIG_COMPAT */ +bool vma_migratable(struct vm_area_struct *vma) +{ + if (vma->vm_flags & (VM_IO | VM_PFNMAP)) + return false; + + /* + * DAX device mappings require predictable access latency, so avoid + * incurring periodic faults. + */ + if (vma_is_dax(vma)) + return false; + + if (is_vm_hugetlb_page(vma) && + !hugepage_migration_supported(hstate_vma(vma))) + return false; + + /* + * Migration allocates pages in the highest zone. If we cannot + * do so then migration (at least from node to node) is not + * possible. + */ + if (vma->vm_file && + gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping)) + < policy_zone) + return false; + return true; +} + struct mempolicy *__get_vma_policy(struct vm_area_struct *vma, unsigned long addr) { @@ -2842,7 +2899,9 @@ int mpol_parse_str(char *str, struct mempolicy **mpol) switch (mode) { case MPOL_PREFERRED: /* - * Insist on a nodelist of one node only + * Insist on a nodelist of one node only, although later + * we use first_node(nodes) to grab a single node, so here + * nodelist (or nodes) cannot be empty. */ if (nodelist) { char *rest = nodelist; @@ -2850,6 +2909,8 @@ int mpol_parse_str(char *str, struct mempolicy **mpol) rest++; if (*rest) goto out; + if (nodes_empty(nodes)) + goto out; } break; case MPOL_INTERLEAVE: diff --git a/mm/migrate.c b/mm/migrate.c index 7605d2c23433..7ded07081be9 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1282,6 +1282,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, int page_was_mapped = 0; struct page *new_hpage; struct anon_vma *anon_vma = NULL; + struct address_space *mapping = NULL; /* * Migratability of hugepages depends on architectures and their size. @@ -1329,18 +1330,36 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, goto put_anon; if (page_mapped(hpage)) { + /* + * try_to_unmap could potentially call huge_pmd_unshare. + * Because of this, take semaphore in write mode here and + * set TTU_RMAP_LOCKED to let lower levels know we have + * taken the lock. + */ + mapping = hugetlb_page_mapping_lock_write(hpage); + if (unlikely(!mapping)) + goto unlock_put_anon; + try_to_unmap(hpage, - TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); + TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS| + TTU_RMAP_LOCKED); page_was_mapped = 1; + /* + * Leave mapping locked until after subsequent call to + * remove_migration_ptes() + */ } if (!page_mapped(hpage)) rc = move_to_new_page(new_hpage, hpage, mode); - if (page_was_mapped) + if (page_was_mapped) { remove_migration_ptes(hpage, - rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false); + rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, true); + i_mmap_unlock_write(mapping); + } +unlock_put_anon: unlock_page(new_hpage); put_anon: diff --git a/mm/mmap.c b/mm/mmap.c index 5f4a6ef458cc..aa58aeb4b44b 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -53,6 +53,9 @@ #include #include +#define CREATE_TRACE_POINTS +#include + #include "internal.h" #ifndef arch_mmap_check @@ -1857,7 +1860,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, return error; } -unsigned long unmapped_area(struct vm_unmapped_area_info *info) +static unsigned long unmapped_area(struct vm_unmapped_area_info *info) { /* * We implement the search by looking for an rbtree node that @@ -1960,7 +1963,7 @@ unsigned long unmapped_area(struct vm_unmapped_area_info *info) return gap_start; } -unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) +static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; @@ -2059,6 +2062,27 @@ unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) return gap_end; } +/* + * Search for an unmapped address range. + * + * We are looking for a range that: + * - does not intersect with any VMA; + * - is contained within the [low_limit, high_limit) interval; + * - is at least the desired size. + * - satisfies (begin_addr & align_mask) == (align_offset & align_mask) + */ +unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) +{ + unsigned long addr; + + if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) + addr = unmapped_area_topdown(info); + else + addr = unmapped_area(info); + + trace_vm_unmapped_area(addr, info); + return addr; +} #ifndef arch_get_mmap_end #define arch_get_mmap_end(addr) (TASK_SIZE) diff --git a/mm/mremap.c b/mm/mremap.c index d28f08a36b96..a7e282ead438 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -133,7 +133,7 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, * such races: * * - During exec() shift_arg_pages(), we use a specially tagged vma - * which rmap call sites look for using is_vma_temporary_stack(). + * which rmap call sites look for using vma_is_temporary_stack(). * * - During mremap(), new_vma is often known to be placed after vma * in rmap traversal order. This ensures rmap will always observe @@ -318,8 +318,8 @@ unsigned long move_page_tables(struct vm_area_struct *vma, static unsigned long move_vma(struct vm_area_struct *vma, unsigned long old_addr, unsigned long old_len, unsigned long new_len, unsigned long new_addr, - bool *locked, struct vm_userfaultfd_ctx *uf, - struct list_head *uf_unmap) + bool *locked, unsigned long flags, + struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap) { struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *new_vma; @@ -408,11 +408,32 @@ static unsigned long move_vma(struct vm_area_struct *vma, if (unlikely(vma->vm_flags & VM_PFNMAP)) untrack_pfn_moved(vma); + if (unlikely(!err && (flags & MREMAP_DONTUNMAP))) { + if (vm_flags & VM_ACCOUNT) { + /* Always put back VM_ACCOUNT since we won't unmap */ + vma->vm_flags |= VM_ACCOUNT; + + vm_acct_memory(vma_pages(new_vma)); + } + + /* We always clear VM_LOCKED[ONFAULT] on the old vma */ + vma->vm_flags &= VM_LOCKED_CLEAR_MASK; + + /* Because we won't unmap we don't need to touch locked_vm */ + goto out; + } + if (do_munmap(mm, old_addr, old_len, uf_unmap) < 0) { /* OOM: unable to split vma, just get accounts right */ vm_unacct_memory(excess >> PAGE_SHIFT); excess = 0; } + + if (vm_flags & VM_LOCKED) { + mm->locked_vm += new_len >> PAGE_SHIFT; + *locked = true; + } +out: mm->hiwater_vm = hiwater_vm; /* Restore VM_ACCOUNT if one or two pieces of vma left */ @@ -422,16 +443,12 @@ static unsigned long move_vma(struct vm_area_struct *vma, vma->vm_next->vm_flags |= VM_ACCOUNT; } - if (vm_flags & VM_LOCKED) { - mm->locked_vm += new_len >> PAGE_SHIFT; - *locked = true; - } - return new_addr; } static struct vm_area_struct *vma_to_resize(unsigned long addr, - unsigned long old_len, unsigned long new_len, unsigned long *p) + unsigned long old_len, unsigned long new_len, unsigned long flags, + unsigned long *p) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma = find_vma(mm, addr); @@ -453,6 +470,10 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr, return ERR_PTR(-EINVAL); } + if (flags & MREMAP_DONTUNMAP && (!vma_is_anonymous(vma) || + vma->vm_flags & VM_SHARED)) + return ERR_PTR(-EINVAL); + if (is_vm_hugetlb_page(vma)) return ERR_PTR(-EINVAL); @@ -497,7 +518,7 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr, static unsigned long mremap_to(unsigned long addr, unsigned long old_len, unsigned long new_addr, unsigned long new_len, bool *locked, - struct vm_userfaultfd_ctx *uf, + unsigned long flags, struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap_early, struct list_head *uf_unmap) { @@ -505,7 +526,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, struct vm_area_struct *vma; unsigned long ret = -EINVAL; unsigned long charged = 0; - unsigned long map_flags; + unsigned long map_flags = 0; if (offset_in_page(new_addr)) goto out; @@ -534,9 +555,11 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, if ((mm->map_count + 2) >= sysctl_max_map_count - 3) return -ENOMEM; - ret = do_munmap(mm, new_addr, new_len, uf_unmap_early); - if (ret) - goto out; + if (flags & MREMAP_FIXED) { + ret = do_munmap(mm, new_addr, new_len, uf_unmap_early); + if (ret) + goto out; + } if (old_len >= new_len) { ret = do_munmap(mm, addr+new_len, old_len - new_len, uf_unmap); @@ -545,13 +568,22 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, old_len = new_len; } - vma = vma_to_resize(addr, old_len, new_len, &charged); + vma = vma_to_resize(addr, old_len, new_len, flags, &charged); if (IS_ERR(vma)) { ret = PTR_ERR(vma); goto out; } - map_flags = MAP_FIXED; + /* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */ + if (flags & MREMAP_DONTUNMAP && + !may_expand_vm(mm, vma->vm_flags, old_len >> PAGE_SHIFT)) { + ret = -ENOMEM; + goto out; + } + + if (flags & MREMAP_FIXED) + map_flags |= MAP_FIXED; + if (vma->vm_flags & VM_MAYSHARE) map_flags |= MAP_SHARED; @@ -561,10 +593,16 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, if (IS_ERR_VALUE(ret)) goto out1; - ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, uf, + /* We got a new mapping */ + if (!(flags & MREMAP_FIXED)) + new_addr = ret; + + ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf, uf_unmap); + if (!(offset_in_page(ret))) goto out; + out1: vm_unacct_memory(charged); @@ -618,12 +656,21 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, */ addr = untagged_addr(addr); - if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE)) + if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP)) return ret; if (flags & MREMAP_FIXED && !(flags & MREMAP_MAYMOVE)) return ret; + /* + * MREMAP_DONTUNMAP is always a move and it does not allow resizing + * in the process. + */ + if (flags & MREMAP_DONTUNMAP && + (!(flags & MREMAP_MAYMOVE) || old_len != new_len)) + return ret; + + if (offset_in_page(addr)) return ret; @@ -641,9 +688,10 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, if (down_write_killable(¤t->mm->mmap_sem)) return -EINTR; - if (flags & MREMAP_FIXED) { + if (flags & (MREMAP_FIXED | MREMAP_DONTUNMAP)) { ret = mremap_to(addr, old_len, new_addr, new_len, - &locked, &uf, &uf_unmap_early, &uf_unmap); + &locked, flags, &uf, &uf_unmap_early, + &uf_unmap); goto out; } @@ -671,7 +719,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, /* * Ok, we need to grow.. */ - vma = vma_to_resize(addr, old_len, new_len, &charged); + vma = vma_to_resize(addr, old_len, new_len, flags, &charged); if (IS_ERR(vma)) { ret = PTR_ERR(vma); goto out; @@ -721,7 +769,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, } ret = move_vma(vma, addr, old_len, new_len, new_addr, - &locked, &uf, &uf_unmap); + &locked, flags, &uf, &uf_unmap); } out: if (offset_in_page(ret)) { diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 2caf780a42e7..7326b54ab728 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2182,12 +2182,12 @@ int write_cache_pages(struct address_space *mapping, if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) range_whole = 1; } - if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) - tag = PAGECACHE_TAG_TOWRITE; - else - tag = PAGECACHE_TAG_DIRTY; - if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) { tag_pages_for_writeback(mapping, index, end); + tag = PAGECACHE_TAG_TOWRITE; + } else { + tag = PAGECACHE_TAG_DIRTY; + } done_index = index; while (!done && (index <= end)) { int i; @@ -2655,7 +2655,7 @@ int clear_page_dirty_for_io(struct page *page) struct address_space *mapping = page_mapping(page); int ret = 0; - BUG_ON(!PageLocked(page)); + VM_BUG_ON_PAGE(!PageLocked(page), page); if (mapping && mapping_cap_account_dirty(mapping)) { struct inode *inode = mapping->host; @@ -2764,7 +2764,7 @@ int test_clear_page_writeback(struct page *page) int __test_set_page_writeback(struct page *page, bool keep_write) { struct address_space *mapping = page_mapping(page); - int ret; + int ret, access_ret; lock_page_memcg(page); if (mapping && mapping_use_writeback_tags(mapping)) { @@ -2807,6 +2807,13 @@ int __test_set_page_writeback(struct page *page, bool keep_write) inc_zone_page_state(page, NR_ZONE_WRITE_PENDING); } unlock_page_memcg(page); + access_ret = arch_make_page_accessible(page); + /* + * If writeback has been triggered on a page that cannot be made + * accessible, it is too late to recover here. + */ + VM_BUG_ON_PAGE(access_ret != 0, page); + return ret; } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 7861567c2c86..6ed27ddfd323 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -95,7 +95,6 @@ DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); */ DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ EXPORT_PER_CPU_SYMBOL(_numa_mem_); -int _node_numa_mem_[MAX_NUMNODES]; #endif /* work_structs for global per-cpu drains */ @@ -701,6 +700,8 @@ void prep_compound_page(struct page *page, unsigned int order) set_compound_head(p, page); } atomic_set(compound_mapcount_ptr(page), -1); + if (hpage_pincount_available(page)) + atomic_set(compound_pincount_ptr(page), 0); } #ifdef CONFIG_DEBUG_PAGEALLOC @@ -803,32 +804,25 @@ static inline void set_page_order(struct page *page, unsigned int order) * * For recording page's order, we use page_private(page). */ -static inline int page_is_buddy(struct page *page, struct page *buddy, +static inline bool page_is_buddy(struct page *page, struct page *buddy, unsigned int order) { - if (page_is_guard(buddy) && page_order(buddy) == order) { - if (page_zone_id(page) != page_zone_id(buddy)) - return 0; + if (!page_is_guard(buddy) && !PageBuddy(buddy)) + return false; - VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); + if (page_order(buddy) != order) + return false; - return 1; - } + /* + * zone check is done late to avoid uselessly calculating + * zone/node ids for pages that could never merge. + */ + if (page_zone_id(page) != page_zone_id(buddy)) + return false; - if (PageBuddy(buddy) && page_order(buddy) == order) { - /* - * zone check is done late to avoid uselessly - * calculating zone/node ids for pages that could - * never merge. - */ - if (page_zone_id(page) != page_zone_id(buddy)) - return 0; + VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); - VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); - - return 1; - } - return 0; + return true; } #ifdef CONFIG_COMPACTION @@ -1164,7 +1158,7 @@ static __always_inline bool free_pages_prepare(struct page *page, if (PageMappingFlags(page)) page->mapping = NULL; if (memcg_kmem_enabled() && PageKmemcg(page)) - __memcg_kmem_uncharge(page, order); + __memcg_kmem_uncharge_page(page, order); if (check_free) bad += free_pages_check(page); if (bad) @@ -3471,8 +3465,7 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, return true; } #endif - if (alloc_harder && - !list_empty(&area->free_list[MIGRATE_HIGHATOMIC])) + if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC)) return true; } return false; @@ -3547,10 +3540,13 @@ static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) static inline unsigned int alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) { - unsigned int alloc_flags = 0; + unsigned int alloc_flags; - if (gfp_mask & __GFP_KSWAPD_RECLAIM) - alloc_flags |= ALLOC_KSWAPD; + /* + * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD + * to save a branch. + */ + alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); #ifdef CONFIG_ZONE_DMA32 if (!zone) @@ -4186,8 +4182,13 @@ gfp_to_alloc_flags(gfp_t gfp_mask) { unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; - /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ + /* + * __GFP_HIGH is assumed to be the same as ALLOC_HIGH + * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD + * to save two branches. + */ BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); + BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); /* * The caller may dip into page reserves a bit more if the caller @@ -4195,7 +4196,8 @@ gfp_to_alloc_flags(gfp_t gfp_mask) * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). */ - alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); + alloc_flags |= (__force int) + (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); if (gfp_mask & __GFP_ATOMIC) { /* @@ -4212,9 +4214,6 @@ gfp_to_alloc_flags(gfp_t gfp_mask) } else if (unlikely(rt_task(current)) && !in_interrupt()) alloc_flags |= ALLOC_HARDER; - if (gfp_mask & __GFP_KSWAPD_RECLAIM) - alloc_flags |= ALLOC_KSWAPD; - #ifdef CONFIG_CMA if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) alloc_flags |= ALLOC_CMA; @@ -4757,14 +4756,13 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, * Restore the original nodemask if it was potentially replaced with * &cpuset_current_mems_allowed to optimize the fast-path attempt. */ - if (unlikely(ac.nodemask != nodemask)) - ac.nodemask = nodemask; + ac.nodemask = nodemask; page = __alloc_pages_slowpath(alloc_mask, order, &ac); out: if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && - unlikely(__memcg_kmem_charge(page, gfp_mask, order) != 0)) { + unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) { __free_pages(page, order); page = NULL; } @@ -7890,8 +7888,8 @@ int __meminit init_per_zone_wmark_min(void) min_free_kbytes = new_min_free_kbytes; if (min_free_kbytes < 128) min_free_kbytes = 128; - if (min_free_kbytes > 65536) - min_free_kbytes = 65536; + if (min_free_kbytes > 262144) + min_free_kbytes = 262144; } else { pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", new_min_free_kbytes, user_min_free_kbytes); @@ -8276,15 +8274,20 @@ struct page *has_unmovable_pages(struct zone *zone, struct page *page, /* * Hugepages are not in LRU lists, but they're movable. + * THPs are on the LRU, but need to be counted as #small pages. * We need not scan over tail pages because we don't * handle each tail page individually in migration. */ - if (PageHuge(page)) { + if (PageHuge(page) || PageTransCompound(page)) { struct page *head = compound_head(page); unsigned int skip_pages; - if (!hugepage_migration_supported(page_hstate(head))) + if (PageHuge(page)) { + if (!hugepage_migration_supported(page_hstate(head))) + return page; + } else if (!PageLRU(head) && !__PageMovable(head)) { return page; + } skip_pages = compound_nr(head) - (page - head); iter += skip_pages - 1; @@ -8425,6 +8428,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, .ignore_skip_hint = true, .no_set_skip_hint = true, .gfp_mask = current_gfp_context(gfp_mask), + .alloc_contig = true, }; INIT_LIST_HEAD(&cc.migratepages); diff --git a/mm/page_counter.c b/mm/page_counter.c index de31470655f6..c56db2d5e159 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -17,29 +17,24 @@ static void propagate_protected_usage(struct page_counter *c, unsigned long usage) { unsigned long protected, old_protected; + unsigned long low, min; long delta; if (!c->parent) return; - if (c->min || atomic_long_read(&c->min_usage)) { - if (usage <= c->min) - protected = usage; - else - protected = 0; - + min = READ_ONCE(c->min); + if (min || atomic_long_read(&c->min_usage)) { + protected = min(usage, min); old_protected = atomic_long_xchg(&c->min_usage, protected); delta = protected - old_protected; if (delta) atomic_long_add(delta, &c->parent->children_min_usage); } - if (c->low || atomic_long_read(&c->low_usage)) { - if (usage <= c->low) - protected = usage; - else - protected = 0; - + low = READ_ONCE(c->low); + if (low || atomic_long_read(&c->low_usage)) { + protected = min(usage, low); old_protected = atomic_long_xchg(&c->low_usage, protected); delta = protected - old_protected; if (delta) @@ -213,7 +208,7 @@ void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages) { struct page_counter *c; - counter->min = nr_pages; + WRITE_ONCE(counter->min, nr_pages); for (c = counter; c; c = c->parent) propagate_protected_usage(c, atomic_long_read(&c->usage)); @@ -230,7 +225,7 @@ void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages) { struct page_counter *c; - counter->low = nr_pages; + WRITE_ONCE(counter->low, nr_pages); for (c = counter; c; c = c->parent) propagate_protected_usage(c, atomic_long_read(&c->usage)); diff --git a/mm/page_ext.c b/mm/page_ext.c index 4ade843ff588..08ded037f89f 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -304,7 +304,7 @@ static int __meminit online_page_ext(unsigned long start_pfn, } for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) { - if (!pfn_present(pfn)) + if (!pfn_in_present_section(pfn)) continue; fail = init_section_page_ext(pfn, nid); } diff --git a/mm/percpu-stats.c b/mm/percpu-stats.c index a5a8b22816ff..32558063c3f9 100644 --- a/mm/percpu-stats.c +++ b/mm/percpu-stats.c @@ -3,7 +3,7 @@ * mm/percpu-debug.c * * Copyright (C) 2017 Facebook Inc. - * Copyright (C) 2017 Dennis Zhou + * Copyright (C) 2017 Dennis Zhou * * Prints statistics about the percpu allocator and backing chunks. */ diff --git a/mm/percpu.c b/mm/percpu.c index e9844086b236..d7e3bc649f4e 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -6,7 +6,7 @@ * Copyright (C) 2009 Tejun Heo * * Copyright (C) 2017 Facebook Inc. - * Copyright (C) 2017 Dennis Zhou + * Copyright (C) 2017 Dennis Zhou * * The percpu allocator handles both static and dynamic areas. Percpu * areas are allocated in chunks which are divided into units. There is diff --git a/mm/rmap.c b/mm/rmap.c index b3e381919835..2df75a119c83 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -22,9 +22,10 @@ * * inode->i_mutex (while writing or truncating, not reading or faulting) * mm->mmap_sem - * page->flags PG_locked (lock_page) + * page->flags PG_locked (lock_page) * (see huegtlbfs below) * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share) * mapping->i_mmap_rwsem + * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) * anon_vma->rwsem * mm->page_table_lock or pte_lock * pgdat->lru_lock (in mark_page_accessed, isolate_lru_page) @@ -43,6 +44,11 @@ * anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon) * ->tasklist_lock * pte map lock + * + * * hugetlbfs PageHuge() pages take locks in this order: + * mapping->i_mmap_rwsem + * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) + * page->flags PG_locked (lock_page) */ #include @@ -1178,6 +1184,9 @@ void page_add_new_anon_rmap(struct page *page, VM_BUG_ON_PAGE(!PageTransHuge(page), page); /* increment count (starts at -1) */ atomic_set(compound_mapcount_ptr(page), 0); + if (hpage_pincount_available(page)) + atomic_set(compound_pincount_ptr(page), 0); + __inc_node_page_state(page, NR_ANON_THPS); } else { /* Anon THP always mapped first with PMD */ @@ -1406,6 +1415,9 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, /* * If sharing is possible, start and end will be adjusted * accordingly. + * + * If called for a huge page, caller must hold i_mmap_rwsem + * in write mode as it is possible to call huge_pmd_unshare. */ adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); @@ -1453,6 +1465,12 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, address = pvmw.address; if (PageHuge(page)) { + /* + * To call huge_pmd_unshare, i_mmap_rwsem must be + * held in write mode. Caller needs to explicitly + * do this outside rmap routines. + */ + VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); if (huge_pmd_unshare(mm, &address, pvmw.pte)) { /* * huge_pmd_unshare unmapped an entire PMD @@ -1696,23 +1714,9 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, return ret; } -bool is_vma_temporary_stack(struct vm_area_struct *vma) -{ - int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); - - if (!maybe_stack) - return false; - - if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == - VM_STACK_INCOMPLETE_SETUP) - return true; - - return false; -} - static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) { - return is_vma_temporary_stack(vma); + return vma_is_temporary_stack(vma); } static int page_mapcount_is_zero(struct page *page) @@ -1974,6 +1978,9 @@ void hugepage_add_new_anon_rmap(struct page *page, { BUG_ON(address < vma->vm_start || address >= vma->vm_end); atomic_set(compound_mapcount_ptr(page), 0); + if (hpage_pincount_available(page)) + atomic_set(compound_pincount_ptr(page), 0); + __page_set_anon_rmap(page, vma, address, 1); } #endif /* CONFIG_HUGETLB_PAGE */ diff --git a/mm/shuffle.c b/mm/shuffle.c index b3fe97fd6654..c716059cbd3c 100644 --- a/mm/shuffle.c +++ b/mm/shuffle.c @@ -72,7 +72,7 @@ static struct page * __meminit shuffle_valid_page(unsigned long pfn, int order) return NULL; /* ...is the pfn in a present section or a hole? */ - if (!pfn_present(pfn)) + if (!pfn_in_present_section(pfn)) return NULL; /* ...is the page free and currently on a free_area list? */ diff --git a/mm/slab.h b/mm/slab.h index 7e94700aa78c..207c83ef6e06 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -348,6 +348,7 @@ static __always_inline int memcg_charge_slab(struct page *page, gfp_t gfp, int order, struct kmem_cache *s) { + unsigned int nr_pages = 1 << order; struct mem_cgroup *memcg; struct lruvec *lruvec; int ret; @@ -360,21 +361,21 @@ static __always_inline int memcg_charge_slab(struct page *page, if (unlikely(!memcg || mem_cgroup_is_root(memcg))) { mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s), - (1 << order)); - percpu_ref_get_many(&s->memcg_params.refcnt, 1 << order); + nr_pages); + percpu_ref_get_many(&s->memcg_params.refcnt, nr_pages); return 0; } - ret = memcg_kmem_charge_memcg(page, gfp, order, memcg); + ret = memcg_kmem_charge(memcg, gfp, nr_pages); if (ret) goto out; lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page)); - mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order); + mod_lruvec_state(lruvec, cache_vmstat_idx(s), nr_pages); /* transer try_charge() page references to kmem_cache */ - percpu_ref_get_many(&s->memcg_params.refcnt, 1 << order); - css_put_many(&memcg->css, 1 << order); + percpu_ref_get_many(&s->memcg_params.refcnt, nr_pages); + css_put_many(&memcg->css, nr_pages); out: css_put(&memcg->css); return ret; @@ -387,6 +388,7 @@ static __always_inline int memcg_charge_slab(struct page *page, static __always_inline void memcg_uncharge_slab(struct page *page, int order, struct kmem_cache *s) { + unsigned int nr_pages = 1 << order; struct mem_cgroup *memcg; struct lruvec *lruvec; @@ -394,15 +396,15 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order, memcg = READ_ONCE(s->memcg_params.memcg); if (likely(!mem_cgroup_is_root(memcg))) { lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page)); - mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order)); - memcg_kmem_uncharge_memcg(page, order, memcg); + mod_lruvec_state(lruvec, cache_vmstat_idx(s), -nr_pages); + memcg_kmem_uncharge(memcg, nr_pages); } else { mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s), - -(1 << order)); + -nr_pages); } rcu_read_unlock(); - percpu_ref_put_many(&s->memcg_params.refcnt, 1 << order); + percpu_ref_put_many(&s->memcg_params.refcnt, nr_pages); } extern void slab_init_memcg_params(struct kmem_cache *); diff --git a/mm/slab_common.c b/mm/slab_common.c index 1907cb2903c7..5282f881d2f5 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1521,7 +1521,7 @@ void dump_unreclaimable_slab(void) mutex_unlock(&slab_mutex); } -#if defined(CONFIG_MEMCG) +#if defined(CONFIG_MEMCG_KMEM) void *memcg_slab_start(struct seq_file *m, loff_t *pos) { struct mem_cgroup *memcg = mem_cgroup_from_seq(m); diff --git a/mm/slub.c b/mm/slub.c index 6589b41d5a60..3098e0cf2899 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -259,7 +259,7 @@ static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr, * freepointer to be restored incorrectly. */ return (void *)((unsigned long)ptr ^ s->random ^ - (unsigned long)kasan_reset_tag((void *)ptr_addr)); + swab((unsigned long)kasan_reset_tag((void *)ptr_addr))); #else return ptr; #endif @@ -2205,11 +2205,11 @@ static void unfreeze_partials(struct kmem_cache *s, struct kmem_cache_node *n = NULL, *n2 = NULL; struct page *page, *discard_page = NULL; - while ((page = c->partial)) { + while ((page = slub_percpu_partial(c))) { struct page new; struct page old; - c->partial = page->next; + slub_set_percpu_partial(c, page); n2 = get_node(s, page_to_nid(page)); if (n != n2) { @@ -2282,7 +2282,7 @@ static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) if (oldpage) { pobjects = oldpage->pobjects; pages = oldpage->pages; - if (drain && pobjects > s->cpu_partial) { + if (drain && pobjects > slub_cpu_partial(s)) { unsigned long flags; /* * partial array is full. Move the existing @@ -2307,7 +2307,7 @@ static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage); - if (unlikely(!s->cpu_partial)) { + if (unlikely(!slub_cpu_partial(s))) { unsigned long flags; local_irq_save(flags); @@ -3512,15 +3512,15 @@ static void set_cpu_partial(struct kmem_cache *s) * 50% to keep some capacity around for frees. */ if (!kmem_cache_has_cpu_partial(s)) - s->cpu_partial = 0; + slub_set_cpu_partial(s, 0); else if (s->size >= PAGE_SIZE) - s->cpu_partial = 2; + slub_set_cpu_partial(s, 2); else if (s->size >= 1024) - s->cpu_partial = 6; + slub_set_cpu_partial(s, 6); else if (s->size >= 256) - s->cpu_partial = 13; + slub_set_cpu_partial(s, 13); else - s->cpu_partial = 30; + slub_set_cpu_partial(s, 30); #endif } @@ -3581,6 +3581,13 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order) */ s->offset = size; size += sizeof(void *); + } else if (size > sizeof(void *)) { + /* + * Store freelist pointer near middle of object to keep + * it away from the edges of the object to avoid small + * sized over/underflows from neighboring allocations. + */ + s->offset = ALIGN(size / 2, sizeof(void *)); } #ifdef CONFIG_SLUB_DEBUG diff --git a/mm/sparse.c b/mm/sparse.c index 65599e8bd636..f1af4d4ee80b 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -664,35 +664,14 @@ static void free_map_bootmem(struct page *memmap) struct page * __meminit populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap) { - struct page *page, *ret; - unsigned long memmap_size = sizeof(struct page) * PAGES_PER_SECTION; - - page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); - if (page) - goto got_map_page; - - ret = vmalloc(memmap_size); - if (ret) - goto got_map_ptr; - - return NULL; -got_map_page: - ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); -got_map_ptr: - - return ret; + return kvmalloc_node(array_size(sizeof(struct page), + PAGES_PER_SECTION), GFP_KERNEL, nid); } static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap) { - struct page *memmap = pfn_to_page(pfn); - - if (is_vmalloc_addr(memmap)) - vfree(memmap); - else - free_pages((unsigned long)memmap, - get_order(sizeof(struct page) * PAGES_PER_SECTION)); + kvfree(pfn_to_page(pfn)); } static void free_map_bootmem(struct page *memmap) @@ -894,7 +873,7 @@ int __meminit sparse_add_section(int nid, unsigned long start_pfn, /* Align memmap to section boundary in the subsection case */ if (section_nr_to_pfn(section_nr) != start_pfn) - memmap = pfn_to_kaddr(section_nr_to_pfn(section_nr)); + memmap = pfn_to_page(section_nr_to_pfn(section_nr)); sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0); return 0; diff --git a/mm/swap.c b/mm/swap.c index cf39d24ada2a..a4af8c999963 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -931,7 +931,6 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, VM_BUG_ON_PAGE(PageLRU(page), page); - SetPageLRU(page); /* * Page becomes evictable in two ways: * 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()]. @@ -958,7 +957,8 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, * looking at the same page) and the evictable page will be stranded * in an unevictable LRU. */ - smp_mb(); + SetPageLRU(page); + smp_mb__after_atomic(); if (page_evictable(page)) { lru = page_lru(page); @@ -986,7 +986,6 @@ void __pagevec_lru_add(struct pagevec *pvec) { pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL); } -EXPORT_SYMBOL(__pagevec_lru_add); /** * pagevec_lookup_entries - gang pagecache lookup diff --git a/mm/swap_slots.c b/mm/swap_slots.c index 63a7b4563a57..0975adc72253 100644 --- a/mm/swap_slots.c +++ b/mm/swap_slots.c @@ -309,7 +309,7 @@ int free_swap_slot(swp_entry_t entry) swp_entry_t get_swap_page(struct page *page) { - swp_entry_t entry, *pentry; + swp_entry_t entry; struct swap_slots_cache *cache; entry.val = 0; @@ -336,13 +336,11 @@ swp_entry_t get_swap_page(struct page *page) if (cache->slots) { repeat: if (cache->nr) { - pentry = &cache->slots[cache->cur++]; - entry = *pentry; - pentry->val = 0; + entry = cache->slots[cache->cur]; + cache->slots[cache->cur++].val = 0; cache->nr--; - } else { - if (refill_swap_slots_cache(cache)) - goto repeat; + } else if (refill_swap_slots_cache(cache)) { + goto repeat; } } mutex_unlock(&cache->alloc_lock); diff --git a/mm/swap_state.c b/mm/swap_state.c index 8e7ce9a9bc5e..ebed37bbf7a3 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -116,7 +116,7 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp) struct address_space *address_space = swap_address_space(entry); pgoff_t idx = swp_offset(entry); XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page)); - unsigned long i, nr = compound_nr(page); + unsigned long i, nr = hpage_nr_pages(page); VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(PageSwapCache(page), page); diff --git a/mm/swapfile.c b/mm/swapfile.c index be33e6176cd9..273a923c275c 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2132,7 +2132,7 @@ int try_to_unuse(unsigned int type, bool frontswap, swp_entry_t entry; unsigned int i; - if (!si->inuse_pages) + if (!READ_ONCE(si->inuse_pages)) return 0; if (!frontswap) @@ -2148,7 +2148,7 @@ int try_to_unuse(unsigned int type, bool frontswap, spin_lock(&mmlist_lock); p = &init_mm.mmlist; - while (si->inuse_pages && + while (READ_ONCE(si->inuse_pages) && !signal_pending(current) && (p = p->next) != &init_mm.mmlist) { @@ -2177,7 +2177,7 @@ int try_to_unuse(unsigned int type, bool frontswap, mmput(prev_mm); i = 0; - while (si->inuse_pages && + while (READ_ONCE(si->inuse_pages) && !signal_pending(current) && (i = find_next_to_unuse(si, i, frontswap)) != 0) { @@ -2219,7 +2219,7 @@ int try_to_unuse(unsigned int type, bool frontswap, * been preempted after get_swap_page(), temporarily hiding that swap. * It's easy and robust (though cpu-intensive) just to keep retrying. */ - if (si->inuse_pages) { + if (READ_ONCE(si->inuse_pages)) { if (!signal_pending(current)) goto retry; retval = -EINTR; @@ -3475,7 +3475,7 @@ int swap_duplicate(swp_entry_t entry) * * Called when allocating swap cache for existing swap entry, * This can return error codes. Returns 0 at success. - * -EBUSY means there is a swap cache. + * -EEXIST means there is a swap cache. * Note: return code is different from swap_duplicate(). */ int swapcache_prepare(swp_entry_t entry) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 1b0d7abad1d4..bd96855f3961 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -276,10 +276,14 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, BUG_ON(dst_addr >= dst_start + len); /* - * Serialize via hugetlb_fault_mutex + * Serialize via i_mmap_rwsem and hugetlb_fault_mutex. + * i_mmap_rwsem ensures the dst_pte remains valid even + * in the case of shared pmds. fault mutex prevents + * races with other faulting threads. */ - idx = linear_page_index(dst_vma, dst_addr); mapping = dst_vma->vm_file->f_mapping; + i_mmap_lock_read(mapping); + idx = linear_page_index(dst_vma, dst_addr); hash = hugetlb_fault_mutex_hash(mapping, idx); mutex_lock(&hugetlb_fault_mutex_table[hash]); @@ -287,6 +291,7 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, dst_pte = huge_pte_alloc(dst_mm, dst_addr, vma_hpagesize); if (!dst_pte) { mutex_unlock(&hugetlb_fault_mutex_table[hash]); + i_mmap_unlock_read(mapping); goto out_unlock; } @@ -294,6 +299,7 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, dst_pteval = huge_ptep_get(dst_pte); if (!huge_pte_none(dst_pteval)) { mutex_unlock(&hugetlb_fault_mutex_table[hash]); + i_mmap_unlock_read(mapping); goto out_unlock; } @@ -301,6 +307,7 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, dst_addr, src_addr, &page); mutex_unlock(&hugetlb_fault_mutex_table[hash]); + i_mmap_unlock_read(mapping); vm_alloc_shared = vm_shared; cond_resched(); diff --git a/mm/vmpressure.c b/mm/vmpressure.c index 4bac22fe1aa2..d69019fc3789 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -280,7 +280,7 @@ void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree, enum vmpressure_levels level; /* For now, no users for root-level efficiency */ - if (!memcg || memcg == root_mem_cgroup) + if (!memcg || mem_cgroup_is_root(memcg)) return; spin_lock(&vmpr->sr_lock); @@ -371,10 +371,8 @@ int vmpressure_register_event(struct mem_cgroup *memcg, int ret = 0; spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL); - if (!spec) { - ret = -ENOMEM; - goto out; - } + if (!spec) + return -ENOMEM; /* Find required level */ token = strsep(&spec, ","); diff --git a/mm/vmscan.c b/mm/vmscan.c index 876370565455..2e8e690d2813 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1084,9 +1084,8 @@ static unsigned long shrink_page_list(struct list_head *page_list, while (!list_empty(page_list)) { struct address_space *mapping; struct page *page; - int may_enter_fs; enum page_references references = PAGEREF_RECLAIM; - bool dirty, writeback; + bool dirty, writeback, may_enter_fs; unsigned int nr_pages; cond_resched(); @@ -1267,7 +1266,7 @@ static unsigned long shrink_page_list(struct list_head *page_list, goto activate_locked_split; } - may_enter_fs = 1; + may_enter_fs = true; /* Adding to swap updated mapping */ mapping = page_mapping(page); @@ -2096,7 +2095,7 @@ static void shrink_active_list(unsigned long nr_to_scan, unsigned long reclaim_pages(struct list_head *page_list) { - int nid = -1; + int nid = NUMA_NO_NODE; unsigned long nr_reclaimed = 0; LIST_HEAD(node_page_list); struct reclaim_stat dummy_stat; @@ -2111,7 +2110,7 @@ unsigned long reclaim_pages(struct list_head *page_list) while (!list_empty(page_list)) { page = lru_to_page(page_list); - if (nid == -1) { + if (nid == NUMA_NO_NODE) { nid = page_to_nid(page); INIT_LIST_HEAD(&node_page_list); } @@ -2132,7 +2131,7 @@ unsigned long reclaim_pages(struct list_head *page_list) putback_lru_page(page); } - nid = -1; + nid = NUMA_NO_NODE; } if (!list_empty(&node_page_list)) { @@ -2427,10 +2426,8 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, case SCAN_FILE: case SCAN_ANON: /* Scan one type exclusively */ - if ((scan_balance == SCAN_FILE) != file) { - lruvec_size = 0; + if ((scan_balance == SCAN_FILE) != file) scan = 0; - } break; default: /* Look ma, no brain */ @@ -3096,7 +3093,6 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, if (sc->memcg_low_skipped) { sc->priority = initial_priority; sc->force_deactivate = 0; - sc->skipped_deactivate = 0; sc->memcg_low_reclaim = 1; sc->memcg_low_skipped = 0; goto retry; @@ -3136,8 +3132,9 @@ static bool allow_direct_reclaim(pg_data_t *pgdat) /* kswapd must be awake if processes are being throttled */ if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) { - pgdat->kswapd_classzone_idx = min(pgdat->kswapd_classzone_idx, - (enum zone_type)ZONE_NORMAL); + if (READ_ONCE(pgdat->kswapd_classzone_idx) > ZONE_NORMAL) + WRITE_ONCE(pgdat->kswapd_classzone_idx, ZONE_NORMAL); + wake_up_interruptible(&pgdat->kswapd_wait); } @@ -3769,9 +3766,9 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx) static enum zone_type kswapd_classzone_idx(pg_data_t *pgdat, enum zone_type prev_classzone_idx) { - if (pgdat->kswapd_classzone_idx == MAX_NR_ZONES) - return prev_classzone_idx; - return pgdat->kswapd_classzone_idx; + enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_classzone_idx); + + return curr_idx == MAX_NR_ZONES ? prev_classzone_idx : curr_idx; } static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, @@ -3815,8 +3812,11 @@ static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_o * the previous request that slept prematurely. */ if (remaining) { - pgdat->kswapd_classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx); - pgdat->kswapd_order = max(pgdat->kswapd_order, reclaim_order); + WRITE_ONCE(pgdat->kswapd_classzone_idx, + kswapd_classzone_idx(pgdat, classzone_idx)); + + if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) + WRITE_ONCE(pgdat->kswapd_order, reclaim_order); } finish_wait(&pgdat->kswapd_wait, &wait); @@ -3893,12 +3893,12 @@ static int kswapd(void *p) tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD; set_freezable(); - pgdat->kswapd_order = 0; - pgdat->kswapd_classzone_idx = MAX_NR_ZONES; + WRITE_ONCE(pgdat->kswapd_order, 0); + WRITE_ONCE(pgdat->kswapd_classzone_idx, MAX_NR_ZONES); for ( ; ; ) { bool ret; - alloc_order = reclaim_order = pgdat->kswapd_order; + alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx); kswapd_try_sleep: @@ -3906,10 +3906,10 @@ static int kswapd(void *p) classzone_idx); /* Read the new order and classzone_idx */ - alloc_order = reclaim_order = pgdat->kswapd_order; + alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx); - pgdat->kswapd_order = 0; - pgdat->kswapd_classzone_idx = MAX_NR_ZONES; + WRITE_ONCE(pgdat->kswapd_order, 0); + WRITE_ONCE(pgdat->kswapd_classzone_idx, MAX_NR_ZONES); ret = try_to_freeze(); if (kthread_should_stop()) @@ -3953,20 +3953,23 @@ void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, enum zone_type classzone_idx) { pg_data_t *pgdat; + enum zone_type curr_idx; if (!managed_zone(zone)) return; if (!cpuset_zone_allowed(zone, gfp_flags)) return; - pgdat = zone->zone_pgdat; - if (pgdat->kswapd_classzone_idx == MAX_NR_ZONES) - pgdat->kswapd_classzone_idx = classzone_idx; - else - pgdat->kswapd_classzone_idx = max(pgdat->kswapd_classzone_idx, - classzone_idx); - pgdat->kswapd_order = max(pgdat->kswapd_order, order); + pgdat = zone->zone_pgdat; + curr_idx = READ_ONCE(pgdat->kswapd_classzone_idx); + + if (curr_idx == MAX_NR_ZONES || curr_idx < classzone_idx) + WRITE_ONCE(pgdat->kswapd_classzone_idx, classzone_idx); + + if (READ_ONCE(pgdat->kswapd_order) < order) + WRITE_ONCE(pgdat->kswapd_order, order); + if (!waitqueue_active(&pgdat->kswapd_wait)) return; @@ -4030,27 +4033,6 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) } #endif /* CONFIG_HIBERNATION */ -/* It's optimal to keep kswapds on the same CPUs as their memory, but - not required for correctness. So if the last cpu in a node goes - away, we get changed to run anywhere: as the first one comes back, - restore their cpu bindings. */ -static int kswapd_cpu_online(unsigned int cpu) -{ - int nid; - - for_each_node_state(nid, N_MEMORY) { - pg_data_t *pgdat = NODE_DATA(nid); - const struct cpumask *mask; - - mask = cpumask_of_node(pgdat->node_id); - - if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids) - /* One of our CPUs online: restore mask */ - set_cpus_allowed_ptr(pgdat->kswapd, mask); - } - return 0; -} - /* * This kswapd start function will be called by init and node-hot-add. * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added. @@ -4090,15 +4072,11 @@ void kswapd_stop(int nid) static int __init kswapd_init(void) { - int nid, ret; + int nid; swap_setup(); for_each_node_state(nid, N_MEMORY) kswapd_run(nid); - ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, - "mm/vmscan:online", kswapd_cpu_online, - NULL); - WARN_ON(ret < 0); return 0; } @@ -4277,29 +4255,6 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) } #endif -/* - * page_evictable - test whether a page is evictable - * @page: the page to test - * - * Test whether page is evictable--i.e., should be placed on active/inactive - * lists vs unevictable list. - * - * Reasons page might not be evictable: - * (1) page's mapping marked unevictable - * (2) page is part of an mlocked VMA - * - */ -int page_evictable(struct page *page) -{ - int ret; - - /* Prevent address_space of inode and swap cache from being freed */ - rcu_read_lock(); - ret = !mapping_unevictable(page_mapping(page)) && !PageMlocked(page); - rcu_read_unlock(); - return ret; -} - /** * check_move_unevictable_pages - check pages for evictability and move to * appropriate zone lru list diff --git a/mm/vmstat.c b/mm/vmstat.c index d0650391c8c1..287a95987b7b 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1171,6 +1171,8 @@ const char * const vmstat_text[] = { "nr_dirtied", "nr_written", "nr_kernel_misc_reclaimable", + "nr_foll_pin_acquired", + "nr_foll_pin_released", /* enum writeback_stat_item counters */ "nr_dirty_threshold", diff --git a/scripts/spelling.txt b/scripts/spelling.txt index ffa838f3a2b5..d9cd24cf0d40 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -177,7 +177,9 @@ atomatically||automatically atomicly||atomically atempt||attempt attachement||attachment +attatch||attach attched||attached +attemp||attempt attemps||attempts attemping||attempting attepmpt||attempt @@ -235,11 +237,13 @@ brievely||briefly brigde||bridge broadcase||broadcast broadcat||broadcast +bufer||buffer bufufer||buffer cacluated||calculated caculate||calculate caculation||calculation cadidate||candidate +cahces||caches calender||calendar calescing||coalescing calle||called @@ -338,7 +342,6 @@ conditon||condition condtion||condition conected||connected conector||connector -connecetd||connected configration||configuration configuartion||configuration configuation||configuration @@ -349,7 +352,9 @@ configuretion||configuration configutation||configuration conider||consider conjuction||conjunction +connecetd||connected connectinos||connections +connetor||connector connnection||connection connnections||connections consistancy||consistency @@ -469,6 +474,7 @@ difinition||definition digial||digital dimention||dimension dimesions||dimensions +disgest||digest dispalying||displaying diplay||display directon||direction @@ -553,6 +559,7 @@ etsablishment||establishment etsbalishment||establishment excecutable||executable exceded||exceeded +exceeed||exceed excellant||excellent execeeded||exceeded execeeds||exceeds @@ -742,6 +749,7 @@ initialzing||initializing initilization||initialization initilize||initialize initliaze||initialize +initilized||initialized inofficial||unofficial inrerface||interface insititute||institute @@ -802,6 +810,7 @@ irrelevent||irrelevant isnt||isn't isssue||issue issus||issues +iteraions||iterations iternations||iterations itertation||iteration itslef||itself @@ -828,6 +837,7 @@ libary||library librairies||libraries libraris||libraries licenceing||licencing +limted||limited logaritmic||logarithmic loggging||logging loggin||login @@ -924,6 +934,7 @@ nerver||never nescessary||necessary nessessary||necessary noticable||noticeable +notication||notification notications||notifications notifcations||notifications notifed||notified @@ -1007,6 +1018,7 @@ pendantic||pedantic peprocessor||preprocessor perfoming||performing perfomring||performing +periperal||peripheral peripherial||peripheral permissons||permissions peroid||period @@ -1043,6 +1055,7 @@ prefferably||preferably premption||preemption prepaired||prepared preperation||preparation +preprare||prepare pressre||pressure primative||primitive princliple||principle @@ -1064,6 +1077,7 @@ processsed||processed processsing||processing procteted||protected prodecure||procedure +progamming||programming progams||programs progess||progress programers||programmers @@ -1151,12 +1165,14 @@ replys||replies reponse||response representaion||representation reqeust||request +reqister||register requestied||requested requiere||require requirment||requirement requred||required requried||required requst||request +requsted||requested reregisteration||reregistration reseting||resetting reseved||reserved @@ -1227,10 +1243,12 @@ seqeuncer||sequencer seqeuencer||sequencer sequece||sequence sequencial||sequential +serivce||service serveral||several servive||service setts||sets settting||setting +shapshot||snapshot shotdown||shutdown shoud||should shouldnt||shouldn't @@ -1328,6 +1346,7 @@ swithcing||switching swithed||switched swithing||switching swtich||switch +syfs||sysfs symetric||symmetric synax||syntax synchonized||synchronized diff --git a/security/integrity/digsig.c b/security/integrity/digsig.c index ea1aae3d07b3..e9cbadade74b 100644 --- a/security/integrity/digsig.c +++ b/security/integrity/digsig.c @@ -6,8 +6,6 @@ * Dmitry Kasatkin */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include diff --git a/security/integrity/digsig_asymmetric.c b/security/integrity/digsig_asymmetric.c index 55aec161d0e1..4e0d6778277e 100644 --- a/security/integrity/digsig_asymmetric.c +++ b/security/integrity/digsig_asymmetric.c @@ -6,8 +6,6 @@ * Dmitry Kasatkin */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index d485f6fc908e..35682852ddea 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -10,8 +10,6 @@ * Using root's kernel master key (kmk), calculate the HMAC */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index 921c8f2afcaf..0632f650958f 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -11,8 +11,6 @@ * evm_inode_removexattr, and evm_verifyxattr */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include diff --git a/security/integrity/evm/evm_secfs.c b/security/integrity/evm/evm_secfs.c index c11c1f7b3ddd..39ad1038d45d 100644 --- a/security/integrity/evm/evm_secfs.c +++ b/security/integrity/evm/evm_secfs.c @@ -10,8 +10,6 @@ * - Get the key and enable EVM */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig index 3f3ee4e2eb0d..edde88dbe576 100644 --- a/security/integrity/ima/Kconfig +++ b/security/integrity/ima/Kconfig @@ -327,3 +327,10 @@ config IMA_QUEUE_EARLY_BOOT_KEYS depends on IMA_MEASURE_ASYMMETRIC_KEYS depends on SYSTEM_TRUSTED_KEYRING default y + +config IMA_SECURE_AND_OR_TRUSTED_BOOT + bool + depends on IMA_ARCH_POLICY + help + This option is selected by architectures to enable secure and/or + trusted boot based on IMA runtime policies. diff --git a/security/integrity/ima/Makefile b/security/integrity/ima/Makefile index 064a256f8725..67dabca670e2 100644 --- a/security/integrity/ima/Makefile +++ b/security/integrity/ima/Makefile @@ -11,6 +11,6 @@ ima-y := ima_fs.o ima_queue.o ima_init.o ima_main.o ima_crypto.o ima_api.o \ ima-$(CONFIG_IMA_APPRAISE) += ima_appraise.o ima-$(CONFIG_IMA_APPRAISE_MODSIG) += ima_modsig.o ima-$(CONFIG_HAVE_IMA_KEXEC) += ima_kexec.o -obj-$(CONFIG_IMA_BLACKLIST_KEYRING) += ima_mok.o -obj-$(CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS) += ima_asymmetric_keys.o -obj-$(CONFIG_IMA_QUEUE_EARLY_BOOT_KEYS) += ima_queue_keys.o +ima-$(CONFIG_IMA_BLACKLIST_KEYRING) += ima_mok.o +ima-$(CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS) += ima_asymmetric_keys.o +ima-$(CONFIG_IMA_QUEUE_EARLY_BOOT_KEYS) += ima_queue_keys.o diff --git a/security/integrity/ima/ima_asymmetric_keys.c b/security/integrity/ima/ima_asymmetric_keys.c index 7678f0e3e84d..aaae80c4e376 100644 --- a/security/integrity/ima/ima_asymmetric_keys.c +++ b/security/integrity/ima/ima_asymmetric_keys.c @@ -9,8 +9,6 @@ * create or update. */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include "ima.h" diff --git a/security/integrity/ima/ima_crypto.c b/security/integrity/ima/ima_crypto.c index 7967a6904851..423c84f95a14 100644 --- a/security/integrity/ima/ima_crypto.c +++ b/security/integrity/ima/ima_crypto.c @@ -10,8 +10,6 @@ * Calculates md5/sha1 file hash, template hash, boot-aggreate hash */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c index 2000e8df0301..a71e822a6e92 100644 --- a/security/integrity/ima/ima_fs.c +++ b/security/integrity/ima/ima_fs.c @@ -12,8 +12,6 @@ * current measurement list and IMA statistics */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include diff --git a/security/integrity/ima/ima_init.c b/security/integrity/ima/ima_init.c index 195cb4079b2b..567468188a61 100644 --- a/security/integrity/ima/ima_init.c +++ b/security/integrity/ima/ima_init.c @@ -11,8 +11,6 @@ * initialization and cleanup functions */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include diff --git a/security/integrity/ima/ima_kexec.c b/security/integrity/ima/ima_kexec.c index 9e94eca48b89..121de3e04af2 100644 --- a/security/integrity/ima/ima_kexec.c +++ b/security/integrity/ima/ima_kexec.c @@ -6,7 +6,6 @@ * Thiago Jung Bauermann * Mimi Zohar */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include #include diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 9fe949c6a530..9d0abedeae77 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -15,8 +15,6 @@ * and ima_file_check. */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include @@ -757,6 +755,9 @@ void process_buffer_measurement(const void *buf, int size, ima_free_template_entry(entry); out: + if (ret < 0) + pr_devel("%s: failed, result: %d\n", __func__, ret); + return; } diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c index 453427048999..c334e0dc6083 100644 --- a/security/integrity/ima/ima_policy.c +++ b/security/integrity/ima/ima_policy.c @@ -7,8 +7,6 @@ * - initialize default measure policy rules */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include diff --git a/security/integrity/ima/ima_queue.c b/security/integrity/ima/ima_queue.c index 1ce8b1701566..8753212ddb18 100644 --- a/security/integrity/ima/ima_queue.c +++ b/security/integrity/ima/ima_queue.c @@ -15,8 +15,6 @@ * ever removed or changed during the boot-cycle. */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include "ima.h" diff --git a/security/integrity/ima/ima_queue_keys.c b/security/integrity/ima/ima_queue_keys.c index c87c72299191..cb3e3f501593 100644 --- a/security/integrity/ima/ima_queue_keys.c +++ b/security/integrity/ima/ima_queue_keys.c @@ -8,8 +8,6 @@ * Enables deferred processing of keys */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include "ima.h" diff --git a/security/integrity/ima/ima_template.c b/security/integrity/ima/ima_template.c index 6aa6408603e3..062d9ad49afb 100644 --- a/security/integrity/ima/ima_template.c +++ b/security/integrity/ima/ima_template.c @@ -9,8 +9,6 @@ * Helpers to manage template descriptors. */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include "ima.h" #include "ima_template_lib.h" diff --git a/security/integrity/ima/ima_template_lib.c b/security/integrity/ima/ima_template_lib.c index 32ae05d88257..9cd1e50f3ccc 100644 --- a/security/integrity/ima/ima_template_lib.c +++ b/security/integrity/ima/ima_template_lib.c @@ -9,8 +9,6 @@ * Library of supported template fields. */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include "ima_template_lib.h" static bool ima_template_hash_algo_allowed(u8 algo) diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 73fc286834d7..298b73794d8b 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -6,6 +6,12 @@ * Mimi Zohar */ +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include #include #include diff --git a/sound/arm/pxa2xx-pcm-lib.c b/sound/arm/pxa2xx-pcm-lib.c index a86c95d89824..e81083e1bc68 100644 --- a/sound/arm/pxa2xx-pcm-lib.c +++ b/sound/arm/pxa2xx-pcm-lib.c @@ -38,7 +38,7 @@ int pxa2xx_pcm_hw_params(struct snd_pcm_substream *substream, struct dma_slave_config config; int ret; - dma_params = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + dma_params = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); if (!dma_params) return 0; @@ -47,7 +47,7 @@ int pxa2xx_pcm_hw_params(struct snd_pcm_substream *substream, return ret; snd_dmaengine_pcm_set_config_from_dai_data(substream, - snd_soc_dai_get_dma_data(rtd->cpu_dai, substream), + snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream), &config); ret = dmaengine_slave_config(chan, &config); @@ -95,7 +95,7 @@ int pxa2xx_pcm_open(struct snd_pcm_substream *substream) runtime->hw = pxa2xx_pcm_hardware; - dma_params = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + dma_params = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); if (!dma_params) return 0; @@ -120,7 +120,7 @@ int pxa2xx_pcm_open(struct snd_pcm_substream *substream) return ret; return snd_dmaengine_pcm_open( - substream, dma_request_slave_channel(rtd->cpu_dai->dev, + substream, dma_request_slave_channel(asoc_rtd_to_cpu(rtd, 0)->dev, dma_params->chan_name)); } EXPORT_SYMBOL(pxa2xx_pcm_open); diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c index 9de1c9a0173e..509290f2efa8 100644 --- a/sound/core/compress_offload.c +++ b/sound/core/compress_offload.c @@ -488,6 +488,48 @@ snd_compr_get_codec_caps(struct snd_compr_stream *stream, unsigned long arg) } #endif /* !COMPR_CODEC_CAPS_OVERFLOW */ +int snd_compr_malloc_pages(struct snd_compr_stream *stream, size_t size) +{ + struct snd_dma_buffer *dmab; + int ret; + + if (snd_BUG_ON(!(stream) || !(stream)->runtime)) + return -EINVAL; + dmab = kzalloc(sizeof(*dmab), GFP_KERNEL); + if (!dmab) + return -ENOMEM; + dmab->dev = stream->dma_buffer.dev; + ret = snd_dma_alloc_pages(dmab->dev.type, dmab->dev.dev, size, dmab); + if (ret < 0) { + kfree(dmab); + return ret; + } + + snd_compr_set_runtime_buffer(stream, dmab); + stream->runtime->dma_bytes = size; + return 1; +} +EXPORT_SYMBOL(snd_compr_malloc_pages); + +int snd_compr_free_pages(struct snd_compr_stream *stream) +{ + struct snd_compr_runtime *runtime = stream->runtime; + + if (snd_BUG_ON(!(stream) || !(stream)->runtime)) + return -EINVAL; + if (runtime->dma_area == NULL) + return 0; + if (runtime->dma_buffer_p != &stream->dma_buffer) { + /* It's a newly allocated buffer. Release it now. */ + snd_dma_free_pages(runtime->dma_buffer_p); + kfree(runtime->dma_buffer_p); + } + + snd_compr_set_runtime_buffer(stream, NULL); + return 0; +} +EXPORT_SYMBOL(snd_compr_free_pages); + /* revisit this with snd_pcm_preallocate_xxx */ static int snd_compr_allocate_buffer(struct snd_compr_stream *stream, struct snd_compr_params *params) diff --git a/sound/core/device.c b/sound/core/device.c index cdc5af526739..bf0b04a7ee79 100644 --- a/sound/core/device.c +++ b/sound/core/device.c @@ -237,3 +237,24 @@ void snd_device_free_all(struct snd_card *card) list_for_each_entry_safe_reverse(dev, next, &card->devices, list) __snd_device_free(dev); } + +/** + * snd_device_get_state - Get the current state of the given device + * @card: the card instance + * @device_data: the data pointer to release + * + * Returns the current state of the given device object. For the valid + * device, either @SNDRV_DEV_BUILD, @SNDRV_DEV_REGISTERED or + * @SNDRV_DEV_DISCONNECTED is returned. + * Or for a non-existing device, -1 is returned as an error. + */ +int snd_device_get_state(struct snd_card *card, void *device_data) +{ + struct snd_device *dev; + + dev = look_for_dev(card, device_data); + if (dev) + return dev->state; + return -1; +} +EXPORT_SYMBOL_GPL(snd_device_get_state); diff --git a/sound/core/info.c b/sound/core/info.c index ca87ae4c30ba..8c6bc5241df5 100644 --- a/sound/core/info.c +++ b/sound/core/info.c @@ -604,7 +604,7 @@ int snd_info_card_free(struct snd_card *card) */ int snd_info_get_line(struct snd_info_buffer *buffer, char *line, int len) { - int c = -1; + int c; if (snd_BUG_ON(!buffer || !buffer->buffer)) return 1; diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c index 13db77771f0f..930def8201f4 100644 --- a/sound/core/oss/pcm_oss.c +++ b/sound/core/oss/pcm_oss.c @@ -884,20 +884,17 @@ static int snd_pcm_oss_change_params_locked(struct snd_pcm_substream *substream) sformat = snd_pcm_plug_slave_format(format, sformat_mask); if ((__force int)sformat < 0 || - !snd_mask_test(sformat_mask, (__force int)sformat)) { - for (sformat = (__force snd_pcm_format_t)0; - (__force int)sformat <= (__force int)SNDRV_PCM_FORMAT_LAST; - sformat = (__force snd_pcm_format_t)((__force int)sformat + 1)) { - if (snd_mask_test(sformat_mask, (__force int)sformat) && + !snd_mask_test_format(sformat_mask, sformat)) { + pcm_for_each_format(sformat) { + if (snd_mask_test_format(sformat_mask, sformat) && snd_pcm_oss_format_to(sformat) >= 0) - break; - } - if ((__force int)sformat > (__force int)SNDRV_PCM_FORMAT_LAST) { - pcm_dbg(substream->pcm, "Cannot find a format!!!\n"); - err = -EINVAL; - goto failure; + goto format_found; } + pcm_dbg(substream->pcm, "Cannot find a format!!!\n"); + err = -EINVAL; + goto failure; } + format_found: err = _snd_pcm_hw_param_set(sparams, SNDRV_PCM_HW_PARAM_FORMAT, (__force int)sformat, 0); if (err < 0) goto failure; @@ -1220,8 +1217,10 @@ snd_pcm_sframes_t snd_pcm_oss_write3(struct snd_pcm_substream *substream, const if (ret < 0) break; } + mutex_unlock(&runtime->oss.params_lock); ret = __snd_pcm_lib_xfer(substream, (void *)ptr, true, frames, in_kernel); + mutex_lock(&runtime->oss.params_lock); if (ret != -EPIPE && ret != -ESTRPIPE) break; /* test, if we can't store new data, because the stream */ @@ -1257,8 +1256,10 @@ snd_pcm_sframes_t snd_pcm_oss_read3(struct snd_pcm_substream *substream, char *p ret = snd_pcm_oss_capture_position_fixup(substream, &delay); if (ret < 0) break; + mutex_unlock(&runtime->oss.params_lock); ret = __snd_pcm_lib_xfer(substream, (void *)ptr, true, frames, in_kernel); + mutex_lock(&runtime->oss.params_lock); if (ret == -EPIPE) { if (runtime->status->state == SNDRV_PCM_STATE_DRAINING) { ret = snd_pcm_kernel_ioctl(substream, SNDRV_PCM_IOCTL_DROP, NULL); diff --git a/sound/core/oss/pcm_plugin.c b/sound/core/oss/pcm_plugin.c index 752d078908e9..fbda4ebf38b3 100644 --- a/sound/core/oss/pcm_plugin.c +++ b/sound/core/oss/pcm_plugin.c @@ -196,82 +196,74 @@ int snd_pcm_plugin_free(struct snd_pcm_plugin *plugin) return 0; } +static snd_pcm_sframes_t calc_dst_frames(struct snd_pcm_substream *plug, + snd_pcm_sframes_t frames) +{ + struct snd_pcm_plugin *plugin, *plugin_next; + + plugin = snd_pcm_plug_first(plug); + while (plugin && frames > 0) { + plugin_next = plugin->next; + if (plugin->dst_frames) { + frames = plugin->dst_frames(plugin, frames); + if (frames < 0) + return frames; + } + if (frames > plugin->buf_frames) + frames = plugin->buf_frames; + plugin = plugin_next; + } + return frames; +} + +static snd_pcm_sframes_t calc_src_frames(struct snd_pcm_substream *plug, + snd_pcm_sframes_t frames) +{ + struct snd_pcm_plugin *plugin, *plugin_prev; + + plugin = snd_pcm_plug_last(plug); + while (plugin && frames > 0) { + if (frames > plugin->buf_frames) + frames = plugin->buf_frames; + plugin_prev = plugin->prev; + if (plugin->src_frames) { + frames = plugin->src_frames(plugin, frames); + if (frames < 0) + return frames; + } + plugin = plugin_prev; + } + return frames; +} + snd_pcm_sframes_t snd_pcm_plug_client_size(struct snd_pcm_substream *plug, snd_pcm_uframes_t drv_frames) { - struct snd_pcm_plugin *plugin, *plugin_prev, *plugin_next; - int stream; - if (snd_BUG_ON(!plug)) return -ENXIO; - if (drv_frames == 0) - return 0; - stream = snd_pcm_plug_stream(plug); - if (stream == SNDRV_PCM_STREAM_PLAYBACK) { - plugin = snd_pcm_plug_last(plug); - while (plugin && drv_frames > 0) { - if (drv_frames > plugin->buf_frames) - drv_frames = plugin->buf_frames; - plugin_prev = plugin->prev; - if (plugin->src_frames) - drv_frames = plugin->src_frames(plugin, drv_frames); - plugin = plugin_prev; - } - } else if (stream == SNDRV_PCM_STREAM_CAPTURE) { - plugin = snd_pcm_plug_first(plug); - while (plugin && drv_frames > 0) { - plugin_next = plugin->next; - if (plugin->dst_frames) - drv_frames = plugin->dst_frames(plugin, drv_frames); - if (drv_frames > plugin->buf_frames) - drv_frames = plugin->buf_frames; - plugin = plugin_next; - } - } else + switch (snd_pcm_plug_stream(plug)) { + case SNDRV_PCM_STREAM_PLAYBACK: + return calc_src_frames(plug, drv_frames); + case SNDRV_PCM_STREAM_CAPTURE: + return calc_dst_frames(plug, drv_frames); + default: snd_BUG(); - return drv_frames; + return -EINVAL; + } } snd_pcm_sframes_t snd_pcm_plug_slave_size(struct snd_pcm_substream *plug, snd_pcm_uframes_t clt_frames) { - struct snd_pcm_plugin *plugin, *plugin_prev, *plugin_next; - snd_pcm_sframes_t frames; - int stream; - if (snd_BUG_ON(!plug)) return -ENXIO; - if (clt_frames == 0) - return 0; - frames = clt_frames; - stream = snd_pcm_plug_stream(plug); - if (stream == SNDRV_PCM_STREAM_PLAYBACK) { - plugin = snd_pcm_plug_first(plug); - while (plugin && frames > 0) { - plugin_next = plugin->next; - if (plugin->dst_frames) { - frames = plugin->dst_frames(plugin, frames); - if (frames < 0) - return frames; - } - if (frames > plugin->buf_frames) - frames = plugin->buf_frames; - plugin = plugin_next; - } - } else if (stream == SNDRV_PCM_STREAM_CAPTURE) { - plugin = snd_pcm_plug_last(plug); - while (plugin) { - if (frames > plugin->buf_frames) - frames = plugin->buf_frames; - plugin_prev = plugin->prev; - if (plugin->src_frames) { - frames = plugin->src_frames(plugin, frames); - if (frames < 0) - return frames; - } - plugin = plugin_prev; - } - } else + switch (snd_pcm_plug_stream(plug)) { + case SNDRV_PCM_STREAM_PLAYBACK: + return calc_dst_frames(plug, clt_frames); + case SNDRV_PCM_STREAM_CAPTURE: + return calc_src_frames(plug, clt_frames); + default: snd_BUG(); - return frames; + return -EINVAL; + } } static int snd_pcm_plug_formats(const struct snd_mask *mask, diff --git a/sound/core/oss/rate.c b/sound/core/oss/rate.c index 7cd09cef6961..d381f4c967c9 100644 --- a/sound/core/oss/rate.c +++ b/sound/core/oss/rate.c @@ -47,7 +47,7 @@ struct rate_priv { unsigned int pos; rate_f func; snd_pcm_sframes_t old_src_frames, old_dst_frames; - struct rate_channel channels[0]; + struct rate_channel channels[]; }; static void rate_init(struct snd_pcm_plugin *plugin) diff --git a/sound/core/pcm.c b/sound/core/pcm.c index a141a301369f..b6d2331a82f7 100644 --- a/sound/core/pcm.c +++ b/sound/core/pcm.c @@ -1019,7 +1019,7 @@ static ssize_t show_pcm_class(struct device *dev, str = "none"; else str = strs[pcm->dev_class]; - return snprintf(buf, PAGE_SIZE, "%s\n", str); + return sprintf(buf, "%s\n", str); } static DEVICE_ATTR(pcm_class, 0444, show_pcm_class, NULL); diff --git a/sound/core/pcm_dmaengine.c b/sound/core/pcm_dmaengine.c index 5749a8a49784..4d059ff2b2e4 100644 --- a/sound/core/pcm_dmaengine.c +++ b/sound/core/pcm_dmaengine.c @@ -240,6 +240,7 @@ EXPORT_SYMBOL_GPL(snd_dmaengine_pcm_pointer_no_residue); snd_pcm_uframes_t snd_dmaengine_pcm_pointer(struct snd_pcm_substream *substream) { struct dmaengine_pcm_runtime_data *prtd = substream_to_prtd(substream); + struct snd_pcm_runtime *runtime = substream->runtime; struct dma_tx_state state; enum dma_status status; unsigned int buf_size; @@ -250,9 +251,12 @@ snd_pcm_uframes_t snd_dmaengine_pcm_pointer(struct snd_pcm_substream *substream) buf_size = snd_pcm_lib_buffer_bytes(substream); if (state.residue > 0 && state.residue <= buf_size) pos = buf_size - state.residue; + + runtime->delay = bytes_to_frames(runtime, + state.in_flight_bytes); } - return bytes_to_frames(substream->runtime, pos); + return bytes_to_frames(runtime, pos); } EXPORT_SYMBOL_GPL(snd_dmaengine_pcm_pointer); @@ -426,7 +430,7 @@ int snd_dmaengine_pcm_refine_runtime_hwparams( * default assumption is that it supports 1, 2 and 4 bytes * widths. */ - for (i = SNDRV_PCM_FORMAT_FIRST; i <= SNDRV_PCM_FORMAT_LAST; i++) { + pcm_for_each_format(i) { int bits = snd_pcm_format_physical_width(i); /* diff --git a/sound/core/pcm_misc.c b/sound/core/pcm_misc.c index a6a541511534..257d412eac5d 100644 --- a/sound/core/pcm_misc.c +++ b/sound/core/pcm_misc.c @@ -42,6 +42,11 @@ struct pcm_format_data { /* we do lots of calculations on snd_pcm_format_t; shut up sparse */ #define INT __force int +static bool valid_format(snd_pcm_format_t format) +{ + return (INT)format >= 0 && (INT)format <= (INT)SNDRV_PCM_FORMAT_LAST; +} + static const struct pcm_format_data pcm_formats[(INT)SNDRV_PCM_FORMAT_LAST+1] = { [SNDRV_PCM_FORMAT_S8] = { .width = 8, .phys = 8, .le = -1, .signd = 1, @@ -259,7 +264,7 @@ static const struct pcm_format_data pcm_formats[(INT)SNDRV_PCM_FORMAT_LAST+1] = int snd_pcm_format_signed(snd_pcm_format_t format) { int val; - if ((INT)format < 0 || (INT)format > (INT)SNDRV_PCM_FORMAT_LAST) + if (!valid_format(format)) return -EINVAL; if ((val = pcm_formats[(INT)format].signd) < 0) return -EINVAL; @@ -307,7 +312,7 @@ EXPORT_SYMBOL(snd_pcm_format_linear); int snd_pcm_format_little_endian(snd_pcm_format_t format) { int val; - if ((INT)format < 0 || (INT)format > (INT)SNDRV_PCM_FORMAT_LAST) + if (!valid_format(format)) return -EINVAL; if ((val = pcm_formats[(INT)format].le) < 0) return -EINVAL; @@ -343,7 +348,7 @@ EXPORT_SYMBOL(snd_pcm_format_big_endian); int snd_pcm_format_width(snd_pcm_format_t format) { int val; - if ((INT)format < 0 || (INT)format > (INT)SNDRV_PCM_FORMAT_LAST) + if (!valid_format(format)) return -EINVAL; if ((val = pcm_formats[(INT)format].width) == 0) return -EINVAL; @@ -361,7 +366,7 @@ EXPORT_SYMBOL(snd_pcm_format_width); int snd_pcm_format_physical_width(snd_pcm_format_t format) { int val; - if ((INT)format < 0 || (INT)format > (INT)SNDRV_PCM_FORMAT_LAST) + if (!valid_format(format)) return -EINVAL; if ((val = pcm_formats[(INT)format].phys) == 0) return -EINVAL; @@ -394,7 +399,7 @@ EXPORT_SYMBOL(snd_pcm_format_size); */ const unsigned char *snd_pcm_format_silence_64(snd_pcm_format_t format) { - if ((INT)format < 0 || (INT)format > (INT)SNDRV_PCM_FORMAT_LAST) + if (!valid_format(format)) return NULL; if (! pcm_formats[(INT)format].phys) return NULL; @@ -418,7 +423,7 @@ int snd_pcm_format_set_silence(snd_pcm_format_t format, void *data, unsigned int unsigned char *dst; const unsigned char *pat; - if ((INT)format < 0 || (INT)format > (INT)SNDRV_PCM_FORMAT_LAST) + if (!valid_format(format)) return -EINVAL; if (samples == 0) return 0; @@ -474,32 +479,32 @@ int snd_pcm_format_set_silence(snd_pcm_format_t format, void *data, unsigned int EXPORT_SYMBOL(snd_pcm_format_set_silence); /** - * snd_pcm_limit_hw_rates - determine rate_min/rate_max fields - * @runtime: the runtime instance + * snd_pcm_hw_limit_rates - determine rate_min/rate_max fields + * @hw: the pcm hw instance * * Determines the rate_min and rate_max fields from the rates bits of - * the given runtime->hw. + * the given hw. * * Return: Zero if successful. */ -int snd_pcm_limit_hw_rates(struct snd_pcm_runtime *runtime) +int snd_pcm_hw_limit_rates(struct snd_pcm_hardware *hw) { int i; for (i = 0; i < (int)snd_pcm_known_rates.count; i++) { - if (runtime->hw.rates & (1 << i)) { - runtime->hw.rate_min = snd_pcm_known_rates.list[i]; + if (hw->rates & (1 << i)) { + hw->rate_min = snd_pcm_known_rates.list[i]; break; } } for (i = (int)snd_pcm_known_rates.count - 1; i >= 0; i--) { - if (runtime->hw.rates & (1 << i)) { - runtime->hw.rate_max = snd_pcm_known_rates.list[i]; + if (hw->rates & (1 << i)) { + hw->rate_max = snd_pcm_known_rates.list[i]; break; } } return 0; } -EXPORT_SYMBOL(snd_pcm_limit_hw_rates); +EXPORT_SYMBOL(snd_pcm_hw_limit_rates); /** * snd_pcm_rate_to_rate_bit - converts sample rate to SNDRV_PCM_RATE_xxx bit diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index cbdf061667fa..aef860256278 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -228,6 +228,9 @@ int snd_pcm_info_user(struct snd_pcm_substream *substream, return err; } +/* macro for simplified cast */ +#define PARAM_MASK_BIT(b) (1U << (__force int)(b)) + static bool hw_support_mmap(struct snd_pcm_substream *substream) { if (!(substream->runtime->hw.info & SNDRV_PCM_INFO_MMAP)) @@ -257,7 +260,7 @@ static int constrain_mask_params(struct snd_pcm_substream *substream, return -EINVAL; /* This parameter is not requested to change by a caller. */ - if (!(params->rmask & (1 << k))) + if (!(params->rmask & PARAM_MASK_BIT(k))) continue; if (trace_hw_mask_param_enabled()) @@ -271,7 +274,7 @@ static int constrain_mask_params(struct snd_pcm_substream *substream, /* Set corresponding flag so that the caller gets it. */ trace_hw_mask_param(substream, k, 0, &old_mask, m); - params->cmask |= 1 << k; + params->cmask |= PARAM_MASK_BIT(k); } return 0; @@ -293,7 +296,7 @@ static int constrain_interval_params(struct snd_pcm_substream *substream, return -EINVAL; /* This parameter is not requested to change by a caller. */ - if (!(params->rmask & (1 << k))) + if (!(params->rmask & PARAM_MASK_BIT(k))) continue; if (trace_hw_interval_param_enabled()) @@ -307,7 +310,7 @@ static int constrain_interval_params(struct snd_pcm_substream *substream, /* Set corresponding flag so that the caller gets it. */ trace_hw_interval_param(substream, k, 0, &old_interval, i); - params->cmask |= 1 << k; + params->cmask |= PARAM_MASK_BIT(k); } return 0; @@ -349,7 +352,7 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, * have 0 so that the parameters are never changed anymore. */ for (k = 0; k <= SNDRV_PCM_HW_PARAM_LAST_INTERVAL; k++) - vstamps[k] = (params->rmask & (1 << k)) ? 1 : 0; + vstamps[k] = (params->rmask & PARAM_MASK_BIT(k)) ? 1 : 0; /* Due to the above design, actual sequence number starts at 2. */ stamp = 2; @@ -417,7 +420,7 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, hw_param_interval(params, r->var)); } - params->cmask |= (1 << r->var); + params->cmask |= PARAM_MASK_BIT(r->var); vstamps[r->var] = stamp; again = true; } @@ -486,9 +489,9 @@ int snd_pcm_hw_refine(struct snd_pcm_substream *substream, params->info = 0; params->fifo_size = 0; - if (params->rmask & (1 << SNDRV_PCM_HW_PARAM_SAMPLE_BITS)) + if (params->rmask & PARAM_MASK_BIT(SNDRV_PCM_HW_PARAM_SAMPLE_BITS)) params->msbits = 0; - if (params->rmask & (1 << SNDRV_PCM_HW_PARAM_RATE)) { + if (params->rmask & PARAM_MASK_BIT(SNDRV_PCM_HW_PARAM_RATE)) { params->rate_num = 0; params->rate_den = 0; } @@ -2293,21 +2296,21 @@ static int snd_pcm_hw_rule_mulkdiv(struct snd_pcm_hw_params *params, static int snd_pcm_hw_rule_format(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) { - unsigned int k; + snd_pcm_format_t k; const struct snd_interval *i = hw_param_interval_c(params, rule->deps[0]); struct snd_mask m; struct snd_mask *mask = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT); snd_mask_any(&m); - for (k = 0; k <= SNDRV_PCM_FORMAT_LAST; ++k) { + pcm_for_each_format(k) { int bits; - if (! snd_mask_test(mask, k)) + if (!snd_mask_test_format(mask, k)) continue; bits = snd_pcm_format_physical_width(k); if (bits <= 0) continue; /* ignore invalid formats */ if ((unsigned)bits < i->min || (unsigned)bits > i->max) - snd_mask_reset(&m, k); + snd_mask_reset(&m, (__force unsigned)k); } return snd_mask_refine(mask, &m); } @@ -2316,14 +2319,15 @@ static int snd_pcm_hw_rule_sample_bits(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) { struct snd_interval t; - unsigned int k; + snd_pcm_format_t k; + t.min = UINT_MAX; t.max = 0; t.openmin = 0; t.openmax = 0; - for (k = 0; k <= SNDRV_PCM_FORMAT_LAST; ++k) { + pcm_for_each_format(k) { int bits; - if (! snd_mask_test(hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT), k)) + if (!snd_mask_test_format(hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT), k)) continue; bits = snd_pcm_format_physical_width(k); if (bits <= 0) @@ -2505,16 +2509,16 @@ static int snd_pcm_hw_constraints_complete(struct snd_pcm_substream *substream) unsigned int mask = 0; if (hw->info & SNDRV_PCM_INFO_INTERLEAVED) - mask |= 1 << SNDRV_PCM_ACCESS_RW_INTERLEAVED; + mask |= PARAM_MASK_BIT(SNDRV_PCM_ACCESS_RW_INTERLEAVED); if (hw->info & SNDRV_PCM_INFO_NONINTERLEAVED) - mask |= 1 << SNDRV_PCM_ACCESS_RW_NONINTERLEAVED; + mask |= PARAM_MASK_BIT(SNDRV_PCM_ACCESS_RW_NONINTERLEAVED); if (hw_support_mmap(substream)) { if (hw->info & SNDRV_PCM_INFO_INTERLEAVED) - mask |= 1 << SNDRV_PCM_ACCESS_MMAP_INTERLEAVED; + mask |= PARAM_MASK_BIT(SNDRV_PCM_ACCESS_MMAP_INTERLEAVED); if (hw->info & SNDRV_PCM_INFO_NONINTERLEAVED) - mask |= 1 << SNDRV_PCM_ACCESS_MMAP_NONINTERLEAVED; + mask |= PARAM_MASK_BIT(SNDRV_PCM_ACCESS_MMAP_NONINTERLEAVED); if (hw->info & SNDRV_PCM_INFO_COMPLEX) - mask |= 1 << SNDRV_PCM_ACCESS_MMAP_COMPLEX; + mask |= PARAM_MASK_BIT(SNDRV_PCM_ACCESS_MMAP_COMPLEX); } err = snd_pcm_hw_constraint_mask(runtime, SNDRV_PCM_HW_PARAM_ACCESS, mask); if (err < 0) @@ -2524,7 +2528,8 @@ static int snd_pcm_hw_constraints_complete(struct snd_pcm_substream *substream) if (err < 0) return err; - err = snd_pcm_hw_constraint_mask(runtime, SNDRV_PCM_HW_PARAM_SUBFORMAT, 1 << SNDRV_PCM_SUBFORMAT_STD); + err = snd_pcm_hw_constraint_mask(runtime, SNDRV_PCM_HW_PARAM_SUBFORMAT, + PARAM_MASK_BIT(SNDRV_PCM_SUBFORMAT_STD)); if (err < 0) return err; diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c index d78a27271d6d..251eaf1152e2 100644 --- a/sound/drivers/aloop.c +++ b/sound/drivers/aloop.c @@ -118,7 +118,7 @@ struct loopback_cable { struct loopback_setup { unsigned int notify: 1; unsigned int rate_shift; - unsigned int format; + snd_pcm_format_t format; unsigned int rate; unsigned int channels; struct snd_ctl_elem_id active_id; @@ -1432,7 +1432,7 @@ static int loopback_format_info(struct snd_kcontrol *kcontrol, uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; uinfo->count = 1; uinfo->value.integer.min = 0; - uinfo->value.integer.max = SNDRV_PCM_FORMAT_LAST; + uinfo->value.integer.max = (__force int)SNDRV_PCM_FORMAT_LAST; uinfo->value.integer.step = 1; return 0; } @@ -1443,7 +1443,7 @@ static int loopback_format_get(struct snd_kcontrol *kcontrol, struct loopback *loopback = snd_kcontrol_chip(kcontrol); ucontrol->value.integer.value[0] = - loopback->setup[kcontrol->id.subdevice] + (__force int)loopback->setup[kcontrol->id.subdevice] [kcontrol->id.device].format; return 0; } diff --git a/sound/drivers/dummy.c b/sound/drivers/dummy.c index 02ac3f4e0c02..b5486de08b97 100644 --- a/sound/drivers/dummy.c +++ b/sound/drivers/dummy.c @@ -901,10 +901,10 @@ static int snd_card_dummy_new_mixer(struct snd_dummy *dummy) static void print_formats(struct snd_dummy *dummy, struct snd_info_buffer *buffer) { - int i; + snd_pcm_format_t i; - for (i = 0; i <= SNDRV_PCM_FORMAT_LAST; i++) { - if (dummy->pcm_hw.formats & (1ULL << i)) + pcm_for_each_format(i) { + if (dummy->pcm_hw.formats & pcm_format_to_bits(i)) snd_iprintf(buffer, " %s", snd_pcm_format_name(i)); } } diff --git a/sound/firewire/bebob/bebob.c b/sound/firewire/bebob/bebob.c index 976d8cb9a34f..2c8e3392a490 100644 --- a/sound/firewire/bebob/bebob.c +++ b/sound/firewire/bebob/bebob.c @@ -509,7 +509,7 @@ MODULE_DEVICE_TABLE(ieee1394, bebob_id_table); static struct fw_driver bebob_driver = { .driver = { .owner = THIS_MODULE, - .name = "snd-bebob", + .name = KBUILD_MODNAME, .bus = &fw_bus_type, }, .probe = bebob_probe, diff --git a/sound/firewire/digi00x/digi00x.c b/sound/firewire/digi00x/digi00x.c index 1f5fc0e7c024..c84b913a9fe0 100644 --- a/sound/firewire/digi00x/digi00x.c +++ b/sound/firewire/digi00x/digi00x.c @@ -192,7 +192,7 @@ MODULE_DEVICE_TABLE(ieee1394, snd_dg00x_id_table); static struct fw_driver dg00x_driver = { .driver = { .owner = THIS_MODULE, - .name = "snd-firewire-digi00x", + .name = KBUILD_MODNAME, .bus = &fw_bus_type, }, .probe = snd_dg00x_probe, diff --git a/sound/firewire/fireface/ff.c b/sound/firewire/fireface/ff.c index f5a016560eb8..b62a4fd22407 100644 --- a/sound/firewire/fireface/ff.c +++ b/sound/firewire/fireface/ff.c @@ -224,7 +224,7 @@ MODULE_DEVICE_TABLE(ieee1394, snd_ff_id_table); static struct fw_driver ff_driver = { .driver = { .owner = THIS_MODULE, - .name = "snd-fireface", + .name = KBUILD_MODNAME, .bus = &fw_bus_type, }, .probe = snd_ff_probe, diff --git a/sound/firewire/fireworks/fireworks.c b/sound/firewire/fireworks/fireworks.c index 134fc9ee26b9..b1cc013a3540 100644 --- a/sound/firewire/fireworks/fireworks.c +++ b/sound/firewire/fireworks/fireworks.c @@ -362,7 +362,7 @@ MODULE_DEVICE_TABLE(ieee1394, efw_id_table); static struct fw_driver efw_driver = { .driver = { .owner = THIS_MODULE, - .name = "snd-fireworks", + .name = KBUILD_MODNAME, .bus = &fw_bus_type, }, .probe = efw_probe, diff --git a/sound/firewire/tascam/tascam-hwdep.c b/sound/firewire/tascam/tascam-hwdep.c index c29a97f6f638..6f38335fe10b 100644 --- a/sound/firewire/tascam/tascam-hwdep.c +++ b/sound/firewire/tascam/tascam-hwdep.c @@ -17,6 +17,7 @@ static long tscm_hwdep_read_locked(struct snd_tscm *tscm, char __user *buf, long count, loff_t *offset) + __releases(&tscm->lock) { struct snd_firewire_event_lock_status event = { .type = SNDRV_FIREWIRE_EVENT_LOCK_STATUS, @@ -36,6 +37,7 @@ static long tscm_hwdep_read_locked(struct snd_tscm *tscm, char __user *buf, static long tscm_hwdep_read_queue(struct snd_tscm *tscm, char __user *buf, long remained, loff_t *offset) + __releases(&tscm->lock) { char __user *pos = buf; unsigned int type = SNDRV_FIREWIRE_EVENT_TASCAM_CONTROL; diff --git a/sound/firewire/tascam/tascam.c b/sound/firewire/tascam/tascam.c index addc464503bc..5dac0d9fc58e 100644 --- a/sound/firewire/tascam/tascam.c +++ b/sound/firewire/tascam/tascam.c @@ -224,7 +224,7 @@ MODULE_DEVICE_TABLE(ieee1394, snd_tscm_id_table); static struct fw_driver tscm_driver = { .driver = { .owner = THIS_MODULE, - .name = "snd-firewire-tascam", + .name = KBUILD_MODNAME, .bus = &fw_bus_type, }, .probe = snd_tscm_probe, diff --git a/sound/hda/hdac_device.c b/sound/hda/hdac_device.c index 9a526aeef8da..e3119f5cb0d5 100644 --- a/sound/hda/hdac_device.c +++ b/sound/hda/hdac_device.c @@ -204,7 +204,7 @@ EXPORT_SYMBOL_GPL(snd_hdac_device_set_chip_name); */ int snd_hdac_codec_modalias(struct hdac_device *codec, char *buf, size_t size) { - return snprintf(buf, size, "hdaudio:v%08Xr%08Xa%02X\n", + return scnprintf(buf, size, "hdaudio:v%08Xr%08Xa%02X\n", codec->vendor_id, codec->revision_id, codec->type); } EXPORT_SYMBOL_GPL(snd_hdac_codec_modalias); diff --git a/sound/isa/sb/emu8000_pcm.c b/sound/isa/sb/emu8000_pcm.c index e377ac93f37f..8e8257c574b0 100644 --- a/sound/isa/sb/emu8000_pcm.c +++ b/sound/isa/sb/emu8000_pcm.c @@ -435,7 +435,7 @@ enum { #define LOOP_WRITE(rec, offset, _buf, count, mode) \ do { \ struct snd_emu8000 *emu = (rec)->emu; \ - unsigned short *buf = (unsigned short *)(_buf); \ + unsigned short *buf = (__force unsigned short *)(_buf); \ snd_emu8000_write_wait(emu, 1); \ EMU8000_SMALW_WRITE(emu, offset); \ while (count > 0) { \ @@ -492,7 +492,7 @@ static int emu8k_pcm_silence(struct snd_pcm_substream *subs, #define LOOP_WRITE(rec, pos, _buf, count, mode) \ do { \ struct snd_emu8000 *emu = rec->emu; \ - unsigned short *buf = (unsigned short *)(_buf); \ + unsigned short *buf = (__force unsigned short *)(_buf); \ snd_emu8000_write_wait(emu, 1); \ EMU8000_SMALW_WRITE(emu, pos + rec->loop_start[0]); \ if (rec->voices > 1) \ diff --git a/sound/pci/ali5451/ali5451.c b/sound/pci/ali5451/ali5451.c index 4f524a9dbbca..4462375d2d82 100644 --- a/sound/pci/ali5451/ali5451.c +++ b/sound/pci/ali5451/ali5451.c @@ -1070,7 +1070,7 @@ static int snd_ali_trigger(struct snd_pcm_substream *substream, { struct snd_ali *codec = snd_pcm_substream_chip(substream); struct snd_pcm_substream *s; - unsigned int what, whati, capture_flag; + unsigned int what, whati; struct snd_ali_voice *pvoice, *evoice; unsigned int val; int do_start; @@ -1088,7 +1088,7 @@ static int snd_ali_trigger(struct snd_pcm_substream *substream, return -EINVAL; } - what = whati = capture_flag = 0; + what = whati = 0; snd_pcm_group_for_each_entry(s, substream) { if ((struct snd_ali *) snd_pcm_substream_chip(s) == codec) { pvoice = s->runtime->private_data; @@ -1110,8 +1110,6 @@ static int snd_ali_trigger(struct snd_pcm_substream *substream, evoice->running = 0; } snd_pcm_trigger_done(s, substream); - if (pvoice->mode) - capture_flag = 1; } } spin_lock(&codec->reg_lock); diff --git a/sound/pci/emu10k1/emu10k1_main.c b/sound/pci/emu10k1/emu10k1_main.c index a89a7e603ca8..6ff581733a19 100644 --- a/sound/pci/emu10k1/emu10k1_main.c +++ b/sound/pci/emu10k1/emu10k1_main.c @@ -1789,6 +1789,7 @@ int snd_emu10k1_create(struct snd_card *card, int idx, err; int is_audigy; size_t page_table_size; + __le32 *pgtbl; unsigned int silent_page; const struct snd_emu_chip_details *c; static const struct snd_device_ops ops = { @@ -2009,8 +2010,9 @@ int snd_emu10k1_create(struct snd_card *card, /* Clear silent pages and set up pointers */ memset(emu->silent_page.area, 0, emu->silent_page.bytes); silent_page = emu->silent_page.addr << emu->address_mode; + pgtbl = (__le32 *)emu->ptb_pages.area; for (idx = 0; idx < (emu->address_mode ? MAXPAGES1 : MAXPAGES0); idx++) - ((u32 *)emu->ptb_pages.area)[idx] = cpu_to_le32(silent_page | idx); + pgtbl[idx] = cpu_to_le32(silent_page | idx); /* set up voice indices */ for (idx = 0; idx < NUM_G; idx++) { diff --git a/sound/pci/hda/Kconfig b/sound/pci/hda/Kconfig index bd48335d09d7..e1d3082a4fe9 100644 --- a/sound/pci/hda/Kconfig +++ b/sound/pci/hda/Kconfig @@ -184,6 +184,7 @@ comment "Set to Y if you want auto-loading the codec driver" config SND_HDA_CODEC_CA0132_DSP bool "Support new DSP code for CA0132 codec" depends on SND_HDA_CODEC_CA0132 + default y select SND_HDA_DSP_LOADER select FW_LOADER help diff --git a/sound/pci/hda/hda_codec.c b/sound/pci/hda/hda_codec.c index 53e7732ef752..a34a2c9f4bcf 100644 --- a/sound/pci/hda/hda_codec.c +++ b/sound/pci/hda/hda_codec.c @@ -88,7 +88,7 @@ struct hda_conn_list { struct list_head list; int len; hda_nid_t nid; - hda_nid_t conns[0]; + hda_nid_t conns[]; }; /* look up the cached results */ diff --git a/sound/pci/hda/hda_controller.c b/sound/pci/hda/hda_controller.c index 2609e391ce54..9765652a73d7 100644 --- a/sound/pci/hda/hda_controller.c +++ b/sound/pci/hda/hda_controller.c @@ -373,7 +373,7 @@ static int azx_get_sync_time(ktime_t *device, u32 wallclk_ctr, wallclk_cycles; bool direction; u32 dma_select; - u32 timeout = 200; + u32 timeout; u32 retry_count = 0; runtime = substream->runtime; diff --git a/sound/pci/hda/patch_ca0132.c b/sound/pci/hda/patch_ca0132.c index ded8bc07d755..34fe753a46fb 100644 --- a/sound/pci/hda/patch_ca0132.c +++ b/sound/pci/hda/patch_ca0132.c @@ -1180,6 +1180,7 @@ static const struct snd_pci_quirk ca0132_quirks[] = { SND_PCI_QUIRK(0x1458, 0xA016, "Recon3Di", QUIRK_R3DI), SND_PCI_QUIRK(0x1458, 0xA026, "Gigabyte G1.Sniper Z97", QUIRK_R3DI), SND_PCI_QUIRK(0x1458, 0xA036, "Gigabyte GA-Z170X-Gaming 7", QUIRK_R3DI), + SND_PCI_QUIRK(0x3842, 0x1038, "EVGA X99 Classified", QUIRK_R3DI), SND_PCI_QUIRK(0x1102, 0x0013, "Recon3D", QUIRK_R3D), SND_PCI_QUIRK(0x1102, 0x0051, "Sound Blaster AE-5", QUIRK_AE5), {} @@ -2698,7 +2699,7 @@ struct dsp_image_seg { u32 magic; u32 chip_addr; u32 count; - u32 data[0]; + u32 data[]; }; static const u32 g_magic_value = 0x4c46584d; diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c index 5119a9ae3d8a..bb287a916dae 100644 --- a/sound/pci/hda/patch_hdmi.c +++ b/sound/pci/hda/patch_hdmi.c @@ -154,7 +154,6 @@ struct hdmi_spec { struct hda_multi_out multiout; struct hda_pcm_stream pcm_playback; - bool use_jack_detect; /* jack detection enabled */ bool use_acomp_notifier; /* use eld_notify callback for hotplug */ bool acomp_registered; /* audio component registered in this driver */ struct drm_audio_component_audio_ops drm_audio_ops; @@ -753,7 +752,7 @@ static void hdmi_setup_audio_infoframe(struct hda_codec *codec, * Unsolicited events */ -static bool hdmi_present_sense(struct hdmi_spec_per_pin *per_pin, int repoll); +static void hdmi_present_sense(struct hdmi_spec_per_pin *per_pin, int repoll); static void check_presence_and_report(struct hda_codec *codec, hda_nid_t nid, int dev_id) @@ -764,8 +763,7 @@ static void check_presence_and_report(struct hda_codec *codec, hda_nid_t nid, if (pin_idx < 0) return; mutex_lock(&spec->pcm_lock); - if (hdmi_present_sense(get_pin(spec, pin_idx), 1)) - snd_hda_jack_report_sync(codec); + hdmi_present_sense(get_pin(spec, pin_idx), 1); mutex_unlock(&spec->pcm_lock); } @@ -779,21 +777,9 @@ static void jack_callback(struct hda_codec *codec, check_presence_and_report(codec, jack->nid, jack->dev_id); } -static void hdmi_intrinsic_event(struct hda_codec *codec, unsigned int res) +static void hdmi_intrinsic_event(struct hda_codec *codec, unsigned int res, + struct hda_jack_tbl *jack) { - int tag = res >> AC_UNSOL_RES_TAG_SHIFT; - struct hda_jack_tbl *jack; - - if (codec->dp_mst) { - int dev_entry = - (res & AC_UNSOL_RES_DE) >> AC_UNSOL_RES_DE_SHIFT; - - jack = snd_hda_jack_tbl_get_from_tag(codec, tag, dev_entry); - } else { - jack = snd_hda_jack_tbl_get_from_tag(codec, tag, 0); - } - if (!jack) - return; jack->jack_dirty = 1; codec_dbg(codec, @@ -853,7 +839,7 @@ static void hdmi_unsol_event(struct hda_codec *codec, unsigned int res) } if (subtag == 0) - hdmi_intrinsic_event(codec, res); + hdmi_intrinsic_event(codec, res, jack); else hdmi_non_intrinsic_event(codec, res); } @@ -1480,21 +1466,60 @@ static void hdmi_pcm_reset_pin(struct hdmi_spec *spec, per_pin->channels = 0; } +static struct snd_jack *pin_idx_to_pcm_jack(struct hda_codec *codec, + struct hdmi_spec_per_pin *per_pin) +{ + struct hdmi_spec *spec = codec->spec; + + if (per_pin->pcm_idx >= 0) + return spec->pcm_rec[per_pin->pcm_idx].jack; + else + return NULL; +} + /* update per_pin ELD from the given new ELD; * setup info frame and notification accordingly + * also notify ELD kctl and report jack status changes */ -static bool update_eld(struct hda_codec *codec, +static void update_eld(struct hda_codec *codec, struct hdmi_spec_per_pin *per_pin, - struct hdmi_eld *eld) + struct hdmi_eld *eld, + int repoll) { struct hdmi_eld *pin_eld = &per_pin->sink_eld; struct hdmi_spec *spec = codec->spec; + struct snd_jack *pcm_jack; bool old_eld_valid = pin_eld->eld_valid; bool eld_changed; int pcm_idx; + if (eld->eld_valid) { + if (eld->eld_size <= 0 || + snd_hdmi_parse_eld(codec, &eld->info, eld->eld_buffer, + eld->eld_size) < 0) { + eld->eld_valid = false; + if (repoll) { + schedule_delayed_work(&per_pin->work, + msecs_to_jiffies(300)); + return; + } + } + } + + if (!eld->eld_valid || eld->eld_size <= 0) { + eld->eld_valid = false; + eld->eld_size = 0; + } + /* for monitor disconnection, save pcm_idx firstly */ pcm_idx = per_pin->pcm_idx; + + /* + * pcm_idx >=0 before update_eld() means it is in monitor + * disconnected event. Jack must be fetched before update_eld(). + */ + pcm_jack = pin_idx_to_pcm_jack(codec, per_pin); + if (spec->dyn_pcm_assign) { if (eld->eld_valid) { hdmi_attach_hda_pcm(spec, per_pin); @@ -1509,6 +1534,8 @@ static bool update_eld(struct hda_codec *codec, */ if (pcm_idx == -1) pcm_idx = per_pin->pcm_idx; + if (!pcm_jack) + pcm_jack = pin_idx_to_pcm_jack(codec, per_pin); if (eld->eld_valid) snd_hdmi_show_eld(codec, &eld->info); @@ -1547,42 +1574,17 @@ static bool update_eld(struct hda_codec *codec, SNDRV_CTL_EVENT_MASK_VALUE | SNDRV_CTL_EVENT_MASK_INFO, &get_hdmi_pcm(spec, pcm_idx)->eld_ctl->id); - return eld_changed; + + if (eld_changed && pcm_jack) + snd_jack_report(pcm_jack, + (eld->monitor_present && eld->eld_valid) ? + SND_JACK_AVOUT : 0); } -static struct snd_jack *pin_idx_to_pcm_jack(struct hda_codec *codec, - struct hdmi_spec_per_pin *per_pin) -{ - struct hdmi_spec *spec = codec->spec; - struct snd_jack *jack = NULL; - struct hda_jack_tbl *jack_tbl; - - /* if !dyn_pcm_assign, get jack from hda_jack_tbl - * in !dyn_pcm_assign case, spec->pcm_rec[].jack is not - * NULL even after snd_hda_jack_tbl_clear() is called to - * free snd_jack. This may cause access invalid memory - * when calling snd_jack_report - */ - if (per_pin->pcm_idx >= 0 && spec->dyn_pcm_assign) { - jack = spec->pcm_rec[per_pin->pcm_idx].jack; - } else if (!spec->dyn_pcm_assign) { - /* - * jack tbl doesn't support DP MST - * DP MST will use dyn_pcm_assign, - * so DP MST will never come here - */ - jack_tbl = snd_hda_jack_tbl_get_mst(codec, per_pin->pin_nid, - per_pin->dev_id); - if (jack_tbl) - jack = jack_tbl->jack; - } - return jack; -} /* update ELD and jack state via HD-audio verbs */ -static bool hdmi_present_sense_via_verbs(struct hdmi_spec_per_pin *per_pin, +static void hdmi_present_sense_via_verbs(struct hdmi_spec_per_pin *per_pin, int repoll) { - struct hda_jack_tbl *jack; struct hda_codec *codec = per_pin->codec; struct hdmi_spec *spec = codec->spec; struct hdmi_eld *eld = &spec->temp_eld; @@ -1597,9 +1599,11 @@ static bool hdmi_present_sense_via_verbs(struct hdmi_spec_per_pin *per_pin, * the unsolicited response to avoid custom WARs. */ int present; - bool ret; - bool do_repoll = false; - struct snd_jack *pcm_jack = NULL; + int ret; + + ret = snd_hda_power_up_pm(codec); + if (ret < 0 && pm_runtime_suspended(hda_codec_dev(codec))) + goto out; present = snd_hda_jack_pin_sense(codec, pin_nid, dev_id); @@ -1618,62 +1622,12 @@ static bool hdmi_present_sense_via_verbs(struct hdmi_spec_per_pin *per_pin, if (spec->ops.pin_get_eld(codec, pin_nid, dev_id, eld->eld_buffer, &eld->eld_size) < 0) eld->eld_valid = false; - else { - if (snd_hdmi_parse_eld(codec, &eld->info, eld->eld_buffer, - eld->eld_size) < 0) - eld->eld_valid = false; - } - if (!eld->eld_valid && repoll) - do_repoll = true; } - if (do_repoll) { - schedule_delayed_work(&per_pin->work, msecs_to_jiffies(300)); - } else { - /* - * pcm_idx >=0 before update_eld() means it is in monitor - * disconnected event. Jack must be fetched before - * update_eld(). - */ - pcm_jack = pin_idx_to_pcm_jack(codec, per_pin); - update_eld(codec, per_pin, eld); - if (!pcm_jack) - pcm_jack = pin_idx_to_pcm_jack(codec, per_pin); - } - - ret = !repoll || !eld->monitor_present || eld->eld_valid; - - jack = snd_hda_jack_tbl_get_mst(codec, pin_nid, per_pin->dev_id); - if (jack) { - jack->block_report = !ret; - jack->pin_sense = (eld->monitor_present && eld->eld_valid) ? - AC_PINSENSE_PRESENCE : 0; - - if (spec->dyn_pcm_assign && pcm_jack && !do_repoll) { - int state = 0; - - if (jack->pin_sense & AC_PINSENSE_PRESENCE) - state = SND_JACK_AVOUT; - snd_jack_report(pcm_jack, state); - } - - /* - * snd_hda_jack_pin_sense() call at the beginning of this - * function, updates jack->pins_sense and clears - * jack->jack_dirty, therefore snd_hda_jack_report_sync() will - * not override the jack->pin_sense. - * - * snd_hda_jack_report_sync() is superfluous for dyn_pcm_assign - * case. The jack->pin_sense update was already performed, and - * hda_jack->jack is NULL for dyn_pcm_assign. - * - * Don't call snd_hda_jack_report_sync() for - * dyn_pcm_assign. - */ - ret = ret && !spec->dyn_pcm_assign; - } + update_eld(codec, per_pin, eld, repoll); mutex_unlock(&per_pin->lock); - return ret; + out: + snd_hda_power_down_pm(codec); } /* update ELD and jack state via audio component */ @@ -1682,64 +1636,25 @@ static void sync_eld_via_acomp(struct hda_codec *codec, { struct hdmi_spec *spec = codec->spec; struct hdmi_eld *eld = &spec->temp_eld; - struct snd_jack *jack = NULL; - bool changed; - int size; mutex_lock(&per_pin->lock); eld->monitor_present = false; - size = snd_hdac_acomp_get_eld(&codec->core, per_pin->pin_nid, + eld->eld_size = snd_hdac_acomp_get_eld(&codec->core, per_pin->pin_nid, per_pin->dev_id, &eld->monitor_present, eld->eld_buffer, ELD_MAX_SIZE); - if (size > 0) { - size = min(size, ELD_MAX_SIZE); - if (snd_hdmi_parse_eld(codec, &eld->info, - eld->eld_buffer, size) < 0) - size = -EINVAL; - } - - if (size > 0) { - eld->eld_valid = true; - eld->eld_size = size; - } else { - eld->eld_valid = false; - eld->eld_size = 0; - } - - /* pcm_idx >=0 before update_eld() means it is in monitor - * disconnected event. Jack must be fetched before update_eld() - */ - jack = pin_idx_to_pcm_jack(codec, per_pin); - changed = update_eld(codec, per_pin, eld); - if (jack == NULL) - jack = pin_idx_to_pcm_jack(codec, per_pin); - if (changed && jack) - snd_jack_report(jack, - (eld->monitor_present && eld->eld_valid) ? - SND_JACK_AVOUT : 0); + eld->eld_valid = (eld->eld_size > 0); + update_eld(codec, per_pin, eld, 0); mutex_unlock(&per_pin->lock); } -static bool hdmi_present_sense(struct hdmi_spec_per_pin *per_pin, int repoll) +static void hdmi_present_sense(struct hdmi_spec_per_pin *per_pin, int repoll) { struct hda_codec *codec = per_pin->codec; - int ret; - /* no temporary power up/down needed for component notifier */ - if (!codec_has_acomp(codec)) { - ret = snd_hda_power_up_pm(codec); - if (ret < 0 && pm_runtime_suspended(hda_codec_dev(codec))) { - snd_hda_power_down_pm(codec); - return false; - } - ret = hdmi_present_sense_via_verbs(per_pin, repoll); - snd_hda_power_down_pm(codec); - } else { + if (!codec_has_acomp(codec)) + hdmi_present_sense_via_verbs(per_pin, repoll); + else sync_eld_via_acomp(codec, per_pin); - ret = false; /* don't call snd_hda_jack_report_sync() */ - } - - return ret; } static void hdmi_repoll_eld(struct work_struct *work) @@ -1759,8 +1674,7 @@ static void hdmi_repoll_eld(struct work_struct *work) per_pin->repoll_count = 0; mutex_lock(&spec->pcm_lock); - if (hdmi_present_sense(per_pin, per_pin->repoll_count)) - snd_hda_jack_report_sync(per_pin->codec); + hdmi_present_sense(per_pin, per_pin->repoll_count); mutex_unlock(&spec->pcm_lock); } @@ -2206,15 +2120,23 @@ static void free_hdmi_jack_priv(struct snd_jack *jack) pcm->jack = NULL; } -static int add_hdmi_jack_kctl(struct hda_codec *codec, - struct hdmi_spec *spec, - int pcm_idx, - const char *name) +static int generic_hdmi_build_jack(struct hda_codec *codec, int pcm_idx) { + char hdmi_str[32] = "HDMI/DP"; + struct hdmi_spec *spec = codec->spec; + struct hdmi_spec_per_pin *per_pin = get_pin(spec, pcm_idx); struct snd_jack *jack; + int pcmdev = get_pcm_rec(spec, pcm_idx)->device; int err; - err = snd_jack_new(codec->card, name, SND_JACK_AVOUT, &jack, + if (pcmdev > 0) + sprintf(hdmi_str + strlen(hdmi_str), ",pcm=%d", pcmdev); + if (!spec->dyn_pcm_assign && + !is_jack_detectable(codec, per_pin->pin_nid)) + strncat(hdmi_str, " Phantom", + sizeof(hdmi_str) - strlen(hdmi_str) - 1); + + err = snd_jack_new(codec->card, hdmi_str, SND_JACK_AVOUT, &jack, true, false); if (err < 0) return err; @@ -2225,48 +2147,6 @@ static int add_hdmi_jack_kctl(struct hda_codec *codec, return 0; } -static int generic_hdmi_build_jack(struct hda_codec *codec, int pcm_idx) -{ - char hdmi_str[32] = "HDMI/DP"; - struct hdmi_spec *spec = codec->spec; - struct hdmi_spec_per_pin *per_pin; - struct hda_jack_tbl *jack; - int pcmdev = get_pcm_rec(spec, pcm_idx)->device; - bool phantom_jack; - int ret; - - if (pcmdev > 0) - sprintf(hdmi_str + strlen(hdmi_str), ",pcm=%d", pcmdev); - - if (spec->dyn_pcm_assign) - return add_hdmi_jack_kctl(codec, spec, pcm_idx, hdmi_str); - - /* for !dyn_pcm_assign, we still use hda_jack for compatibility */ - /* if !dyn_pcm_assign, it must be non-MST mode. - * This means pcms and pins are statically mapped. - * And pcm_idx is pin_idx. - */ - per_pin = get_pin(spec, pcm_idx); - phantom_jack = !is_jack_detectable(codec, per_pin->pin_nid); - if (phantom_jack) - strncat(hdmi_str, " Phantom", - sizeof(hdmi_str) - strlen(hdmi_str) - 1); - ret = snd_hda_jack_add_kctl_mst(codec, per_pin->pin_nid, - per_pin->dev_id, hdmi_str, phantom_jack, - 0, NULL); - if (ret < 0) - return ret; - jack = snd_hda_jack_tbl_get_mst(codec, per_pin->pin_nid, - per_pin->dev_id); - if (jack == NULL) - return 0; - /* assign jack->jack to pcm_rec[].jack to - * align with dyn_pcm_assign mode - */ - spec->pcm_rec[pcm_idx].jack = jack->jack; - return 0; -} - static int generic_hdmi_build_controls(struct hda_codec *codec) { struct hdmi_spec *spec = codec->spec; @@ -2355,7 +2235,6 @@ static int generic_hdmi_init(struct hda_codec *codec) int pin_idx; mutex_lock(&spec->bind_lock); - spec->use_jack_detect = !codec->jackpoll_interval; for (pin_idx = 0; pin_idx < spec->num_pins; pin_idx++) { struct hdmi_spec_per_pin *per_pin = get_pin(spec, pin_idx); hda_nid_t pin_nid = per_pin->pin_nid; @@ -2365,12 +2244,8 @@ static int generic_hdmi_init(struct hda_codec *codec) hdmi_init_pin(codec, pin_nid); if (codec_has_acomp(codec)) continue; - if (spec->use_jack_detect) - snd_hda_jack_detect_enable(codec, pin_nid, dev_id); - else - snd_hda_jack_detect_enable_callback_mst(codec, pin_nid, - dev_id, - jack_callback); + snd_hda_jack_detect_enable_callback_mst(codec, pin_nid, dev_id, + jack_callback); } mutex_unlock(&spec->bind_lock); return 0; @@ -2532,12 +2407,6 @@ static void reprogram_jack_detect(struct hda_codec *codec, hda_nid_t nid, unsigned int val = use_acomp ? 0 : (AC_USRSP_EN | tbl->tag); snd_hda_codec_write_cache(codec, nid, 0, AC_VERB_SET_UNSOLICITED_ENABLE, val); - } else { - /* if no jack entry was defined beforehand, create a new one - * at need (i.e. only when notifier is cleared) - */ - if (!use_acomp) - snd_hda_jack_detect_enable(codec, nid, dev_id); } } @@ -2553,13 +2422,11 @@ static void generic_acomp_notifier_set(struct drm_audio_component *acomp, spec->use_acomp_notifier = use_acomp; spec->codec->relaxed_resume = use_acomp; /* reprogram each jack detection logic depending on the notifier */ - if (spec->use_jack_detect) { - for (i = 0; i < spec->num_pins; i++) - reprogram_jack_detect(spec->codec, - get_pin(spec, i)->pin_nid, - get_pin(spec, i)->dev_id, - use_acomp); - } + for (i = 0; i < spec->num_pins; i++) + reprogram_jack_detect(spec->codec, + get_pin(spec, i)->pin_nid, + get_pin(spec, i)->dev_id, + use_acomp); mutex_unlock(&spec->bind_lock); } diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 63e1a56f705b..f66a48154a57 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -107,6 +107,7 @@ struct alc_spec { unsigned int done_hp_init:1; unsigned int no_shutup_pins:1; unsigned int ultra_low_power:1; + unsigned int has_hs_key:1; /* for PLL fix */ hda_nid_t pll_nid; @@ -367,7 +368,9 @@ static void alc_fill_eapd_coef(struct hda_codec *codec) case 0x10ec0215: case 0x10ec0233: case 0x10ec0235: + case 0x10ec0236: case 0x10ec0255: + case 0x10ec0256: case 0x10ec0257: case 0x10ec0282: case 0x10ec0283: @@ -379,11 +382,6 @@ static void alc_fill_eapd_coef(struct hda_codec *codec) case 0x10ec0300: alc_update_coef_idx(codec, 0x10, 1<<9, 0); break; - case 0x10ec0236: - case 0x10ec0256: - alc_write_coef_idx(codec, 0x36, 0x5757); - alc_update_coef_idx(codec, 0x10, 1<<9, 0); - break; case 0x10ec0275: alc_update_coef_idx(codec, 0xe, 0, 1<<0); break; @@ -2982,6 +2980,107 @@ static int alc269_parse_auto_config(struct hda_codec *codec) return alc_parse_auto_config(codec, alc269_ignore, ssids); } +static const struct hda_jack_keymap alc_headset_btn_keymap[] = { + { SND_JACK_BTN_0, KEY_PLAYPAUSE }, + { SND_JACK_BTN_1, KEY_VOICECOMMAND }, + { SND_JACK_BTN_2, KEY_VOLUMEUP }, + { SND_JACK_BTN_3, KEY_VOLUMEDOWN }, + {} +}; + +static void alc_headset_btn_callback(struct hda_codec *codec, + struct hda_jack_callback *jack) +{ + int report = 0; + + if (jack->unsol_res & (7 << 13)) + report |= SND_JACK_BTN_0; + + if (jack->unsol_res & (1 << 16 | 3 << 8)) + report |= SND_JACK_BTN_1; + + /* Volume up key */ + if (jack->unsol_res & (7 << 23)) + report |= SND_JACK_BTN_2; + + /* Volume down key */ + if (jack->unsol_res & (7 << 10)) + report |= SND_JACK_BTN_3; + + jack->jack->button_state = report; +} + +static void alc_disable_headset_jack_key(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + + if (!spec->has_hs_key) + return; + + switch (codec->core.vendor_id) { + case 0x10ec0215: + case 0x10ec0225: + case 0x10ec0285: + case 0x10ec0295: + case 0x10ec0289: + case 0x10ec0299: + alc_write_coef_idx(codec, 0x48, 0x0); + alc_update_coef_idx(codec, 0x49, 0x0045, 0x0); + alc_update_coef_idx(codec, 0x44, 0x0045 << 8, 0x0); + break; + case 0x10ec0236: + case 0x10ec0256: + alc_write_coef_idx(codec, 0x48, 0x0); + alc_update_coef_idx(codec, 0x49, 0x0045, 0x0); + break; + } +} + +static void alc_enable_headset_jack_key(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + + if (!spec->has_hs_key) + return; + + switch (codec->core.vendor_id) { + case 0x10ec0215: + case 0x10ec0225: + case 0x10ec0285: + case 0x10ec0295: + case 0x10ec0289: + case 0x10ec0299: + alc_write_coef_idx(codec, 0x48, 0xd011); + alc_update_coef_idx(codec, 0x49, 0x007f, 0x0045); + alc_update_coef_idx(codec, 0x44, 0x007f << 8, 0x0045 << 8); + break; + case 0x10ec0236: + case 0x10ec0256: + alc_write_coef_idx(codec, 0x48, 0xd011); + alc_update_coef_idx(codec, 0x49, 0x007f, 0x0045); + break; + } +} + +static void alc_fixup_headset_jack(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + struct alc_spec *spec = codec->spec; + + switch (action) { + case HDA_FIXUP_ACT_PRE_PROBE: + spec->has_hs_key = 1; + snd_hda_jack_detect_enable_callback(codec, 0x55, + alc_headset_btn_callback); + snd_hda_jack_add_kctl(codec, 0x55, "Headset Jack", false, + SND_JACK_HEADSET, alc_headset_btn_keymap); + break; + case HDA_FIXUP_ACT_INIT: + alc_enable_headset_jack_key(codec); + break; + } +} + static void alc269vb_toggle_power_output(struct hda_codec *codec, int power_up) { alc_update_coef_idx(codec, 0x04, 1 << 11, power_up ? (1 << 11) : 0); @@ -3269,7 +3368,13 @@ static void alc256_init(struct hda_codec *codec) alc_update_coefex_idx(codec, 0x57, 0x04, 0x0007, 0x4); /* Hight power */ alc_update_coefex_idx(codec, 0x53, 0x02, 0x8000, 1 << 15); /* Clear bit */ alc_update_coefex_idx(codec, 0x53, 0x02, 0x8000, 0 << 15); - alc_update_coef_idx(codec, 0x36, 1 << 13, 1 << 5); /* Switch pcbeep path to Line in path*/ + /* + * Expose headphone mic (or possibly Line In on some machines) instead + * of PC Beep on 1Ah, and disable 1Ah loopback for all outputs. See + * Documentation/sound/hd-audio/realtek-pc-beep.rst for details of + * this register. + */ + alc_write_coef_idx(codec, 0x36, 0x5757); } static void alc256_shutup(struct hda_codec *codec) @@ -3372,6 +3477,8 @@ static void alc225_shutup(struct hda_codec *codec) if (!hp_pin) hp_pin = 0x21; + + alc_disable_headset_jack_key(codec); /* 3k pull low control for Headset jack. */ alc_update_coef_idx(codec, 0x4a, 0, 3 << 10); @@ -3411,6 +3518,9 @@ static void alc225_shutup(struct hda_codec *codec) alc_update_coef_idx(codec, 0x4a, 3<<4, 2<<4); msleep(30); } + + alc_update_coef_idx(codec, 0x4a, 3 << 10, 0); + alc_enable_headset_jack_key(codec); } static void alc_default_init(struct hda_codec *codec) @@ -4008,6 +4118,12 @@ static void alc269_fixup_hp_gpio_led(struct hda_codec *codec, alc_fixup_hp_gpio_led(codec, action, 0x08, 0x10); } +static void alc285_fixup_hp_gpio_led(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + alc_fixup_hp_gpio_led(codec, action, 0x04, 0x00); +} + static void alc286_fixup_hp_gpio_led(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -5375,17 +5491,6 @@ static void alc271_hp_gate_mic_jack(struct hda_codec *codec, } } -static void alc256_fixup_dell_xps_13_headphone_noise2(struct hda_codec *codec, - const struct hda_fixup *fix, - int action) -{ - if (action != HDA_FIXUP_ACT_PRE_PROBE) - return; - - snd_hda_codec_amp_stereo(codec, 0x1a, HDA_INPUT, 0, HDA_AMP_VOLMASK, 1); - snd_hda_override_wcaps(codec, 0x1a, get_wcaps(codec, 0x1a) & ~AC_WCAP_IN_AMP); -} - static void alc269_fixup_limit_int_mic_boost(struct hda_codec *codec, const struct hda_fixup *fix, int action) @@ -5662,69 +5767,6 @@ static void alc285_fixup_invalidate_dacs(struct hda_codec *codec, snd_hda_override_wcaps(codec, 0x03, 0); } -static const struct hda_jack_keymap alc_headset_btn_keymap[] = { - { SND_JACK_BTN_0, KEY_PLAYPAUSE }, - { SND_JACK_BTN_1, KEY_VOICECOMMAND }, - { SND_JACK_BTN_2, KEY_VOLUMEUP }, - { SND_JACK_BTN_3, KEY_VOLUMEDOWN }, - {} -}; - -static void alc_headset_btn_callback(struct hda_codec *codec, - struct hda_jack_callback *jack) -{ - int report = 0; - - if (jack->unsol_res & (7 << 13)) - report |= SND_JACK_BTN_0; - - if (jack->unsol_res & (1 << 16 | 3 << 8)) - report |= SND_JACK_BTN_1; - - /* Volume up key */ - if (jack->unsol_res & (7 << 23)) - report |= SND_JACK_BTN_2; - - /* Volume down key */ - if (jack->unsol_res & (7 << 10)) - report |= SND_JACK_BTN_3; - - jack->jack->button_state = report; -} - -static void alc_fixup_headset_jack(struct hda_codec *codec, - const struct hda_fixup *fix, int action) -{ - - switch (action) { - case HDA_FIXUP_ACT_PRE_PROBE: - snd_hda_jack_detect_enable_callback(codec, 0x55, - alc_headset_btn_callback); - snd_hda_jack_add_kctl(codec, 0x55, "Headset Jack", false, - SND_JACK_HEADSET, alc_headset_btn_keymap); - break; - case HDA_FIXUP_ACT_INIT: - switch (codec->core.vendor_id) { - case 0x10ec0215: - case 0x10ec0225: - case 0x10ec0285: - case 0x10ec0295: - case 0x10ec0289: - case 0x10ec0299: - alc_write_coef_idx(codec, 0x48, 0xd011); - alc_update_coef_idx(codec, 0x49, 0x007f, 0x0045); - alc_update_coef_idx(codec, 0x44, 0x007f << 8, 0x0045 << 8); - break; - case 0x10ec0236: - case 0x10ec0256: - alc_write_coef_idx(codec, 0x48, 0xd011); - alc_update_coef_idx(codec, 0x49, 0x007f, 0x0045); - break; - } - break; - } -} - static void alc295_fixup_chromebook(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -5863,8 +5905,6 @@ enum { ALC298_FIXUP_DELL1_MIC_NO_PRESENCE, ALC298_FIXUP_DELL_AIO_MIC_NO_PRESENCE, ALC275_FIXUP_DELL_XPS, - ALC256_FIXUP_DELL_XPS_13_HEADPHONE_NOISE, - ALC256_FIXUP_DELL_XPS_13_HEADPHONE_NOISE2, ALC293_FIXUP_LENOVO_SPK_NOISE, ALC233_FIXUP_LENOVO_LINE2_MIC_HOTKEY, ALC255_FIXUP_DELL_SPK_NOISE, @@ -5923,6 +5963,7 @@ enum { ALC294_FIXUP_ASUS_DUAL_SPK, ALC285_FIXUP_THINKPAD_HEADSET_JACK, ALC294_FIXUP_ASUS_HPE, + ALC285_FIXUP_HP_GPIO_LED, }; static const struct hda_fixup alc269_fixups[] = { @@ -6604,23 +6645,6 @@ static const struct hda_fixup alc269_fixups[] = { {} } }, - [ALC256_FIXUP_DELL_XPS_13_HEADPHONE_NOISE] = { - .type = HDA_FIXUP_VERBS, - .v.verbs = (const struct hda_verb[]) { - /* Disable pass-through path for FRONT 14h */ - {0x20, AC_VERB_SET_COEF_INDEX, 0x36}, - {0x20, AC_VERB_SET_PROC_COEF, 0x1737}, - {} - }, - .chained = true, - .chain_id = ALC255_FIXUP_DELL1_MIC_NO_PRESENCE - }, - [ALC256_FIXUP_DELL_XPS_13_HEADPHONE_NOISE2] = { - .type = HDA_FIXUP_FUNC, - .v.func = alc256_fixup_dell_xps_13_headphone_noise2, - .chained = true, - .chain_id = ALC256_FIXUP_DELL_XPS_13_HEADPHONE_NOISE - }, [ALC293_FIXUP_LENOVO_SPK_NOISE] = { .type = HDA_FIXUP_FUNC, .v.func = alc_fixup_disable_aamix, @@ -7061,6 +7085,10 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC294_FIXUP_ASUS_HEADSET_MIC }, + [ALC285_FIXUP_HP_GPIO_LED] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc285_fixup_hp_gpio_led, + }, }; static const struct snd_pci_quirk alc269_fixup_tbl[] = { @@ -7114,17 +7142,14 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1028, 0x06de, "Dell", ALC293_FIXUP_DISABLE_AAMIX_MULTIJACK), SND_PCI_QUIRK(0x1028, 0x06df, "Dell", ALC293_FIXUP_DISABLE_AAMIX_MULTIJACK), SND_PCI_QUIRK(0x1028, 0x06e0, "Dell", ALC293_FIXUP_DISABLE_AAMIX_MULTIJACK), - SND_PCI_QUIRK(0x1028, 0x0704, "Dell XPS 13 9350", ALC256_FIXUP_DELL_XPS_13_HEADPHONE_NOISE2), SND_PCI_QUIRK(0x1028, 0x0706, "Dell Inspiron 7559", ALC256_FIXUP_DELL_INSPIRON_7559_SUBWOOFER), SND_PCI_QUIRK(0x1028, 0x0725, "Dell Inspiron 3162", ALC255_FIXUP_DELL_SPK_NOISE), SND_PCI_QUIRK(0x1028, 0x0738, "Dell Precision 5820", ALC269_FIXUP_NO_SHUTUP), - SND_PCI_QUIRK(0x1028, 0x075b, "Dell XPS 13 9360", ALC256_FIXUP_DELL_XPS_13_HEADPHONE_NOISE2), SND_PCI_QUIRK(0x1028, 0x075c, "Dell XPS 27 7760", ALC298_FIXUP_SPK_VOLUME), SND_PCI_QUIRK(0x1028, 0x075d, "Dell AIO", ALC298_FIXUP_SPK_VOLUME), SND_PCI_QUIRK(0x1028, 0x07b0, "Dell Precision 7520", ALC295_FIXUP_DISABLE_DAC3), SND_PCI_QUIRK(0x1028, 0x0798, "Dell Inspiron 17 7000 Gaming", ALC256_FIXUP_DELL_INSPIRON_7559_SUBWOOFER), SND_PCI_QUIRK(0x1028, 0x080c, "Dell WYSE", ALC225_FIXUP_DELL_WYSE_MIC_NO_PRESENCE), - SND_PCI_QUIRK(0x1028, 0x082a, "Dell XPS 13 9360", ALC256_FIXUP_DELL_XPS_13_HEADPHONE_NOISE2), SND_PCI_QUIRK(0x1028, 0x084b, "Dell", ALC274_FIXUP_DELL_AIO_LINEOUT_VERB), SND_PCI_QUIRK(0x1028, 0x084e, "Dell", ALC274_FIXUP_DELL_AIO_LINEOUT_VERB), SND_PCI_QUIRK(0x1028, 0x0871, "Dell Precision 3630", ALC255_FIXUP_DELL_HEADSET_MIC), @@ -7208,6 +7233,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x83b9, "HP Spectre x360", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x103c, 0x8497, "HP Envy x360", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x103c, 0x84e7, "HP Pavilion 15", ALC269_FIXUP_HP_MUTE_LED_MIC3), + SND_PCI_QUIRK(0x103c, 0x8736, "HP", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x1043, 0x103e, "ASUS X540SA", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x103f, "ASUS TX300", ALC282_FIXUP_ASUS_TX300), SND_PCI_QUIRK(0x1043, 0x106d, "Asus K53BE", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), @@ -7477,7 +7503,6 @@ static const struct hda_model_fixup alc269_fixup_models[] = { {.id = ALC298_FIXUP_DELL1_MIC_NO_PRESENCE, .name = "alc298-dell1"}, {.id = ALC298_FIXUP_DELL_AIO_MIC_NO_PRESENCE, .name = "alc298-dell-aio"}, {.id = ALC275_FIXUP_DELL_XPS, .name = "alc275-dell-xps"}, - {.id = ALC256_FIXUP_DELL_XPS_13_HEADPHONE_NOISE, .name = "alc256-dell-xps13"}, {.id = ALC293_FIXUP_LENOVO_SPK_NOISE, .name = "lenovo-spk-noise"}, {.id = ALC233_FIXUP_LENOVO_LINE2_MIC_HOTKEY, .name = "lenovo-hotkey"}, {.id = ALC255_FIXUP_DELL_SPK_NOISE, .name = "dell-spk-noise"}, diff --git a/sound/pci/korg1212/korg1212.c b/sound/pci/korg1212/korg1212.c index 21ab9cc50c71..65a887b217ee 100644 --- a/sound/pci/korg1212/korg1212.c +++ b/sound/pci/korg1212/korg1212.c @@ -30,7 +30,7 @@ #if K1212_DEBUG_LEVEL > 0 #define K1212_DEBUG_PRINTK(fmt,args...) printk(KERN_DEBUG fmt,##args) #else -#define K1212_DEBUG_PRINTK(fmt,...) +#define K1212_DEBUG_PRINTK(fmt,...) do { } while (0) #endif #if K1212_DEBUG_LEVEL > 1 #define K1212_DEBUG_PRINTK_VERBOSE(fmt,args...) printk(KERN_DEBUG fmt,##args) diff --git a/sound/pci/rme9652/hdsp.c b/sound/pci/rme9652/hdsp.c index cc06f0a1a7e4..227aece17e39 100644 --- a/sound/pci/rme9652/hdsp.c +++ b/sound/pci/rme9652/hdsp.c @@ -3353,7 +3353,8 @@ snd_hdsp_proc_read(struct snd_info_entry *entry, struct snd_info_buffer *buffer) return; } } else { - int err = -EINVAL; + int err; + err = hdsp_request_fw_loader(hdsp); if (err < 0) { snd_iprintf(buffer, diff --git a/sound/pci/via82xx.c b/sound/pci/via82xx.c index 799789c8eea9..8b03e2dc503f 100644 --- a/sound/pci/via82xx.c +++ b/sound/pci/via82xx.c @@ -414,6 +414,7 @@ static int build_via_table(struct viadev *dev, struct snd_pcm_substream *substre { unsigned int i, idx, ofs, rest; struct via82xx *chip = snd_pcm_substream_chip(substream); + __le32 *pgtbl; if (dev->table.area == NULL) { /* the start of each lists must be aligned to 8 bytes, @@ -435,6 +436,7 @@ static int build_via_table(struct viadev *dev, struct snd_pcm_substream *substre /* fill the entries */ idx = 0; ofs = 0; + pgtbl = (__le32 *)dev->table.area; for (i = 0; i < periods; i++) { rest = fragsize; /* fill descriptors for a period. @@ -451,7 +453,7 @@ static int build_via_table(struct viadev *dev, struct snd_pcm_substream *substre return -EINVAL; } addr = snd_pcm_sgbuf_get_addr(substream, ofs); - ((u32 *)dev->table.area)[idx << 1] = cpu_to_le32(addr); + pgtbl[idx << 1] = cpu_to_le32(addr); r = snd_pcm_sgbuf_get_chunk_size(substream, ofs, rest); rest -= r; if (! rest) { @@ -466,7 +468,7 @@ static int build_via_table(struct viadev *dev, struct snd_pcm_substream *substre "tbl %d: at %d size %d (rest %d)\n", idx, ofs, r, rest); */ - ((u32 *)dev->table.area)[(idx<<1) + 1] = cpu_to_le32(r | flag); + pgtbl[(idx<<1) + 1] = cpu_to_le32(r | flag); dev->idx_table[idx].offset = ofs; dev->idx_table[idx].size = r; ofs += r; diff --git a/sound/pci/via82xx_modem.c b/sound/pci/via82xx_modem.c index 84e589803e2e..607b7100db1c 100644 --- a/sound/pci/via82xx_modem.c +++ b/sound/pci/via82xx_modem.c @@ -267,6 +267,7 @@ static int build_via_table(struct viadev *dev, struct snd_pcm_substream *substre { unsigned int i, idx, ofs, rest; struct via82xx_modem *chip = snd_pcm_substream_chip(substream); + __le32 *pgtbl; if (dev->table.area == NULL) { /* the start of each lists must be aligned to 8 bytes, @@ -288,6 +289,7 @@ static int build_via_table(struct viadev *dev, struct snd_pcm_substream *substre /* fill the entries */ idx = 0; ofs = 0; + pgtbl = (__le32 *)dev->table.area; for (i = 0; i < periods; i++) { rest = fragsize; /* fill descriptors for a period. @@ -304,7 +306,7 @@ static int build_via_table(struct viadev *dev, struct snd_pcm_substream *substre return -EINVAL; } addr = snd_pcm_sgbuf_get_addr(substream, ofs); - ((u32 *)dev->table.area)[idx << 1] = cpu_to_le32(addr); + pgtbl[idx << 1] = cpu_to_le32(addr); r = PAGE_SIZE - (ofs % PAGE_SIZE); if (rest < r) r = rest; @@ -321,7 +323,7 @@ static int build_via_table(struct viadev *dev, struct snd_pcm_substream *substre "tbl %d: at %d size %d (rest %d)\n", idx, ofs, r, rest); */ - ((u32 *)dev->table.area)[(idx<<1) + 1] = cpu_to_le32(r | flag); + pgtbl[(idx<<1) + 1] = cpu_to_le32(r | flag); dev->idx_table[idx].offset = ofs; dev->idx_table[idx].size = r; ofs += r; diff --git a/sound/ppc/keywest.c b/sound/ppc/keywest.c index 093806d735c6..9554a0c506af 100644 --- a/sound/ppc/keywest.c +++ b/sound/ppc/keywest.c @@ -40,6 +40,7 @@ static int keywest_probe(struct i2c_client *client, static int keywest_attach_adapter(struct i2c_adapter *adapter) { struct i2c_board_info info; + struct i2c_client *client; if (! keywest_ctx) return -EINVAL; @@ -50,9 +51,11 @@ static int keywest_attach_adapter(struct i2c_adapter *adapter) memset(&info, 0, sizeof(struct i2c_board_info)); strlcpy(info.type, "keywest", I2C_NAME_SIZE); info.addr = keywest_ctx->addr; - keywest_ctx->client = i2c_new_device(adapter, &info); - if (!keywest_ctx->client) - return -ENODEV; + client = i2c_new_client_device(adapter, &info); + if (IS_ERR(client)) + return PTR_ERR(client); + keywest_ctx->client = client; + /* * We know the driver is already loaded, so the device should be * already bound. If not it means binding failed, and then there diff --git a/sound/soc/amd/Kconfig b/sound/soc/amd/Kconfig index 5f40517717c4..bce4cee5cb54 100644 --- a/sound/soc/amd/Kconfig +++ b/sound/soc/amd/Kconfig @@ -26,3 +26,13 @@ config SND_SOC_AMD_ACP3x depends on X86 && PCI help This option enables ACP v3.x I2S support on AMD platform + +config SND_SOC_AMD_RV_RT5682_MACH + tristate "AMD RV support for RT5682" + select SND_SOC_RT5682 + select SND_SOC_MAX98357A + select SND_SOC_CROS_EC_CODEC + select I2C_CROS_EC_TUNNEL + depends on SND_SOC_AMD_ACP3x && I2C && CROS_EC + help + This option enables machine driver for RT5682 and MAX9835. diff --git a/sound/soc/amd/Makefile b/sound/soc/amd/Makefile index c4ddc6adb6f0..e6f3d9b469f3 100644 --- a/sound/soc/amd/Makefile +++ b/sound/soc/amd/Makefile @@ -2,8 +2,10 @@ acp_audio_dma-objs := acp-pcm-dma.o snd-soc-acp-da7219mx98357-mach-objs := acp-da7219-max98357a.o snd-soc-acp-rt5645-mach-objs := acp-rt5645.o +snd-soc-acp-rt5682-mach-objs := acp3x-rt5682-max9836.o obj-$(CONFIG_SND_SOC_AMD_ACP) += acp_audio_dma.o obj-$(CONFIG_SND_SOC_AMD_CZ_DA7219MX98357_MACH) += snd-soc-acp-da7219mx98357-mach.o obj-$(CONFIG_SND_SOC_AMD_CZ_RT5645_MACH) += snd-soc-acp-rt5645-mach.o obj-$(CONFIG_SND_SOC_AMD_ACP3x) += raven/ +obj-$(CONFIG_SND_SOC_AMD_RV_RT5682_MACH) += snd-soc-acp-rt5682-mach.o diff --git a/sound/soc/amd/acp-da7219-max98357a.c b/sound/soc/amd/acp-da7219-max98357a.c index 7a5621e5e233..9414d7269c4f 100644 --- a/sound/soc/amd/acp-da7219-max98357a.c +++ b/sound/soc/amd/acp-da7219-max98357a.c @@ -54,7 +54,7 @@ static int cz_da7219_init(struct snd_soc_pcm_runtime *rtd) { int ret; struct snd_soc_card *card = rtd->card; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_component *component = codec_dai->component; dev_info(rtd->dev, "codec dai name = %s\n", codec_dai->name); diff --git a/sound/soc/amd/acp-rt5645.c b/sound/soc/amd/acp-rt5645.c index 91abeb92b648..73b31f88a6b5 100644 --- a/sound/soc/amd/acp-rt5645.c +++ b/sound/soc/amd/acp-rt5645.c @@ -48,7 +48,7 @@ static int cz_aif1_hw_params(struct snd_pcm_substream *substream, { int ret = 0; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); ret = snd_soc_dai_set_pll(codec_dai, 0, RT5645_PLL1_S_MCLK, CZ_PLAT_CLK, params_rate(params) * 512); @@ -73,7 +73,7 @@ static int cz_init(struct snd_soc_pcm_runtime *rtd) struct snd_soc_card *card; struct snd_soc_component *codec; - codec = rtd->codec_dai->component; + codec = asoc_rtd_to_codec(rtd, 0)->component; card = rtd->card; ret = snd_soc_card_jack_new(card, "Headset Jack", diff --git a/sound/soc/amd/acp3x-rt5682-max9836.c b/sound/soc/amd/acp3x-rt5682-max9836.c new file mode 100644 index 000000000000..024a7ee54cd5 --- /dev/null +++ b/sound/soc/amd/acp3x-rt5682-max9836.c @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: GPL-2.0+ +// +// Machine driver for AMD ACP Audio engine using DA7219 & MAX98357 codec. +// +//Copyright 2016 Advanced Micro Devices, Inc. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "raven/acp3x.h" +#include "../codecs/rt5682.h" + +#define PCO_PLAT_CLK 48000000 +#define RT5682_PLL_FREQ (48000 * 512) +#define DUAL_CHANNEL 2 + +static struct snd_soc_jack pco_jack; +static struct clk *rt5682_dai_wclk; +static struct clk *rt5682_dai_bclk; +static struct gpio_desc *dmic_sel; + +static int acp3x_5682_init(struct snd_soc_pcm_runtime *rtd) +{ + int ret; + struct snd_soc_card *card = rtd->card; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_component *component = codec_dai->component; + + dev_info(rtd->dev, "codec dai name = %s\n", codec_dai->name); + + /* set rt5682 dai fmt */ + ret = snd_soc_dai_set_fmt(codec_dai, SND_SOC_DAIFMT_I2S + | SND_SOC_DAIFMT_NB_NF + | SND_SOC_DAIFMT_CBM_CFM); + if (ret < 0) { + dev_err(rtd->card->dev, + "Failed to set rt5682 dai fmt: %d\n", ret); + return ret; + } + + /* set codec PLL */ + ret = snd_soc_dai_set_pll(codec_dai, RT5682_PLL2, RT5682_PLL2_S_MCLK, + PCO_PLAT_CLK, RT5682_PLL_FREQ); + if (ret < 0) { + dev_err(rtd->dev, "can't set rt5682 PLL: %d\n", ret); + return ret; + } + + /* Set codec sysclk */ + ret = snd_soc_dai_set_sysclk(codec_dai, RT5682_SCLK_S_PLL2, + RT5682_PLL_FREQ, SND_SOC_CLOCK_IN); + if (ret < 0) { + dev_err(rtd->dev, + "Failed to set rt5682 SYSCLK: %d\n", ret); + return ret; + } + + /* Set tdm/i2s1 master bclk ratio */ + ret = snd_soc_dai_set_bclk_ratio(codec_dai, 64); + if (ret < 0) { + dev_err(rtd->dev, + "Failed to set rt5682 tdm bclk ratio: %d\n", ret); + return ret; + } + + rt5682_dai_wclk = clk_get(component->dev, "rt5682-dai-wclk"); + rt5682_dai_bclk = clk_get(component->dev, "rt5682-dai-bclk"); + + ret = snd_soc_card_jack_new(card, "Headset Jack", + SND_JACK_HEADSET | SND_JACK_LINEOUT | + SND_JACK_BTN_0 | SND_JACK_BTN_1 | + SND_JACK_BTN_2 | SND_JACK_BTN_3, + &pco_jack, NULL, 0); + if (ret) { + dev_err(card->dev, "HP jack creation failed %d\n", ret); + return ret; + } + + snd_jack_set_key(pco_jack.jack, SND_JACK_BTN_0, KEY_PLAYPAUSE); + snd_jack_set_key(pco_jack.jack, SND_JACK_BTN_1, KEY_VOLUMEUP); + snd_jack_set_key(pco_jack.jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN); + snd_jack_set_key(pco_jack.jack, SND_JACK_BTN_3, KEY_VOICECOMMAND); + + ret = snd_soc_component_set_jack(component, &pco_jack, NULL); + if (ret) { + dev_err(rtd->dev, "Headset Jack call-back failed: %d\n", ret); + return ret; + } + + return ret; +} + +static int rt5682_clk_enable(struct snd_pcm_substream *substream) +{ + int ret = 0; + struct snd_soc_pcm_runtime *rtd = substream->private_data; + + /* RT5682 will support only 48K output with 48M mclk */ + clk_set_rate(rt5682_dai_wclk, 48000); + clk_set_rate(rt5682_dai_bclk, 48000 * 64); + ret = clk_prepare_enable(rt5682_dai_wclk); + if (ret < 0) { + dev_err(rtd->dev, "can't enable wclk %d\n", ret); + return ret; + } + + return ret; +} + +static void rt5682_clk_disable(void) +{ + clk_disable_unprepare(rt5682_dai_wclk); +} + +static const unsigned int channels[] = { + DUAL_CHANNEL, +}; + +static const unsigned int rates[] = { + 48000, +}; + +static const struct snd_pcm_hw_constraint_list constraints_rates = { + .count = ARRAY_SIZE(rates), + .list = rates, + .mask = 0, +}; + +static const struct snd_pcm_hw_constraint_list constraints_channels = { + .count = ARRAY_SIZE(channels), + .list = channels, + .mask = 0, +}; + +static int acp3x_5682_startup(struct snd_pcm_substream *substream) +{ + struct snd_pcm_runtime *runtime = substream->runtime; + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_card *card = rtd->card; + struct acp3x_platform_info *machine = snd_soc_card_get_drvdata(card); + + machine->play_i2s_instance = I2S_SP_INSTANCE; + machine->cap_i2s_instance = I2S_SP_INSTANCE; + + runtime->hw.channels_max = DUAL_CHANNEL; + snd_pcm_hw_constraint_list(runtime, 0, SNDRV_PCM_HW_PARAM_CHANNELS, + &constraints_channels); + snd_pcm_hw_constraint_list(runtime, 0, SNDRV_PCM_HW_PARAM_RATE, + &constraints_rates); + return rt5682_clk_enable(substream); +} + +static int acp3x_max_startup(struct snd_pcm_substream *substream) +{ + struct snd_pcm_runtime *runtime = substream->runtime; + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_card *card = rtd->card; + struct acp3x_platform_info *machine = snd_soc_card_get_drvdata(card); + + machine->play_i2s_instance = I2S_BT_INSTANCE; + + runtime->hw.channels_max = DUAL_CHANNEL; + snd_pcm_hw_constraint_list(runtime, 0, SNDRV_PCM_HW_PARAM_CHANNELS, + &constraints_channels); + snd_pcm_hw_constraint_list(runtime, 0, SNDRV_PCM_HW_PARAM_RATE, + &constraints_rates); + return rt5682_clk_enable(substream); +} + +static int acp3x_ec_dmic0_startup(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_card *card = rtd->card; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct acp3x_platform_info *machine = snd_soc_card_get_drvdata(card); + + machine->cap_i2s_instance = I2S_BT_INSTANCE; + snd_soc_dai_set_bclk_ratio(codec_dai, 64); + if (dmic_sel) + gpiod_set_value(dmic_sel, 0); + + return rt5682_clk_enable(substream); +} + +static int acp3x_ec_dmic1_startup(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_card *card = rtd->card; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct acp3x_platform_info *machine = snd_soc_card_get_drvdata(card); + + machine->cap_i2s_instance = I2S_BT_INSTANCE; + snd_soc_dai_set_bclk_ratio(codec_dai, 64); + if (dmic_sel) + gpiod_set_value(dmic_sel, 1); + + return rt5682_clk_enable(substream); +} + +static void rt5682_shutdown(struct snd_pcm_substream *substream) +{ + rt5682_clk_disable(); +} + +static const struct snd_soc_ops acp3x_5682_ops = { + .startup = acp3x_5682_startup, + .shutdown = rt5682_shutdown, +}; + +static const struct snd_soc_ops acp3x_max_play_ops = { + .startup = acp3x_max_startup, + .shutdown = rt5682_shutdown, +}; + +static const struct snd_soc_ops acp3x_ec_cap0_ops = { + .startup = acp3x_ec_dmic0_startup, + .shutdown = rt5682_shutdown, +}; + +static const struct snd_soc_ops acp3x_ec_cap1_ops = { + .startup = acp3x_ec_dmic1_startup, + .shutdown = rt5682_shutdown, +}; + +SND_SOC_DAILINK_DEF(acp3x_i2s, + DAILINK_COMP_ARRAY(COMP_CPU("acp3x_i2s_playcap.0"))); +SND_SOC_DAILINK_DEF(acp3x_bt, + DAILINK_COMP_ARRAY(COMP_CPU("acp3x_i2s_playcap.2"))); + +SND_SOC_DAILINK_DEF(rt5682, + DAILINK_COMP_ARRAY(COMP_CODEC("i2c-10EC5682:00", "rt5682-aif1"))); +SND_SOC_DAILINK_DEF(max, + DAILINK_COMP_ARRAY(COMP_CODEC("MX98357A:00", "HiFi"))); +SND_SOC_DAILINK_DEF(cros_ec, + DAILINK_COMP_ARRAY(COMP_CODEC("GOOG0013:00", "EC Codec I2S RX"))); + +SND_SOC_DAILINK_DEF(platform, + DAILINK_COMP_ARRAY(COMP_PLATFORM("acp3x_rv_i2s_dma.0"))); + +static struct snd_soc_dai_link acp3x_dai_5682_98357[] = { + { + .name = "acp3x-5682-play", + .stream_name = "Playback", + .dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF + | SND_SOC_DAIFMT_CBM_CFM, + .init = acp3x_5682_init, + .dpcm_playback = 1, + .dpcm_capture = 1, + .ops = &acp3x_5682_ops, + SND_SOC_DAILINK_REG(acp3x_i2s, rt5682, platform), + }, + { + .name = "acp3x-max98357-play", + .stream_name = "HiFi Playback", + .dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF + | SND_SOC_DAIFMT_CBM_CFM, + .dpcm_playback = 1, + .ops = &acp3x_max_play_ops, + SND_SOC_DAILINK_REG(acp3x_bt, max, platform), + }, + { + .name = "acp3x-ec-dmic0-capture", + .stream_name = "Capture DMIC0", + .dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF + | SND_SOC_DAIFMT_CBS_CFS, + .dpcm_capture = 1, + .ops = &acp3x_ec_cap0_ops, + SND_SOC_DAILINK_REG(acp3x_bt, cros_ec, platform), + }, + { + .name = "acp3x-ec-dmic1-capture", + .stream_name = "Capture DMIC1", + .dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF + | SND_SOC_DAIFMT_CBS_CFS, + .dpcm_capture = 1, + .ops = &acp3x_ec_cap1_ops, + SND_SOC_DAILINK_REG(acp3x_bt, cros_ec, platform), + }, +}; + +static const struct snd_soc_dapm_widget acp3x_widgets[] = { + SND_SOC_DAPM_HP("Headphone Jack", NULL), + SND_SOC_DAPM_SPK("Spk", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), +}; + +static const struct snd_soc_dapm_route acp3x_audio_route[] = { + {"Headphone Jack", NULL, "HPOL"}, + {"Headphone Jack", NULL, "HPOR"}, + {"IN1P", NULL, "Headset Mic"}, + {"Spk", NULL, "Speaker"}, +}; + +static const struct snd_kcontrol_new acp3x_mc_controls[] = { + SOC_DAPM_PIN_SWITCH("Headphone Jack"), + SOC_DAPM_PIN_SWITCH("Spk"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), +}; + +static struct snd_soc_card acp3x_card = { + .name = "acp3xalc5682m98357", + .owner = THIS_MODULE, + .dai_link = acp3x_dai_5682_98357, + .num_links = ARRAY_SIZE(acp3x_dai_5682_98357), + .dapm_widgets = acp3x_widgets, + .num_dapm_widgets = ARRAY_SIZE(acp3x_widgets), + .dapm_routes = acp3x_audio_route, + .num_dapm_routes = ARRAY_SIZE(acp3x_audio_route), + .controls = acp3x_mc_controls, + .num_controls = ARRAY_SIZE(acp3x_mc_controls), +}; + +static int acp3x_probe(struct platform_device *pdev) +{ + int ret; + struct snd_soc_card *card; + struct acp3x_platform_info *machine; + + machine = devm_kzalloc(&pdev->dev, sizeof(*machine), GFP_KERNEL); + if (!machine) + return -ENOMEM; + + card = &acp3x_card; + acp3x_card.dev = &pdev->dev; + platform_set_drvdata(pdev, card); + snd_soc_card_set_drvdata(card, machine); + + dmic_sel = devm_gpiod_get(&pdev->dev, "dmic", GPIOD_OUT_LOW); + if (IS_ERR(dmic_sel)) { + dev_err(&pdev->dev, "DMIC gpio failed err=%ld\n", + PTR_ERR(dmic_sel)); + return PTR_ERR(dmic_sel); + } + + ret = devm_snd_soc_register_card(&pdev->dev, &acp3x_card); + if (ret) { + dev_err(&pdev->dev, + "devm_snd_soc_register_card(%s) failed: %d\n", + acp3x_card.name, ret); + return ret; + } + return 0; +} + +static const struct acpi_device_id acp3x_audio_acpi_match[] = { + { "AMDI5682", 0 }, + {}, +}; +MODULE_DEVICE_TABLE(acpi, acp3x_audio_acpi_match); + +static struct platform_driver acp3x_audio = { + .driver = { + .name = "acp3x-alc5682-max98357", + .acpi_match_table = ACPI_PTR(acp3x_audio_acpi_match), + .pm = &snd_soc_pm_ops, + }, + .probe = acp3x_probe, +}; + +module_platform_driver(acp3x_audio); + +MODULE_AUTHOR("akshu.agrawal@amd.com"); +MODULE_DESCRIPTION("ALC5682 & MAX98357 audio support"); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/amd/raven/acp3x-i2s.c b/sound/soc/amd/raven/acp3x-i2s.c index 91a388184e52..3a3c47e820ab 100644 --- a/sound/soc/amd/raven/acp3x-i2s.c +++ b/sound/soc/amd/raven/acp3x-i2s.c @@ -42,7 +42,7 @@ static int acp3x_i2s_set_tdm_slot(struct snd_soc_dai *cpu_dai, u32 tx_mask, u32 rx_mask, int slots, int slot_width) { struct i2s_dev_data *adata; - u32 val, reg_val, frmt_reg, frm_len; + u32 frm_len; u16 slot_len; adata = snd_soc_dai_get_drvdata(cpu_dai); @@ -64,36 +64,7 @@ static int acp3x_i2s_set_tdm_slot(struct snd_soc_dai *cpu_dai, default: return -EINVAL; } - - /* Enable I2S/BT channels TDM, respective TX/RX frame lengths.*/ - frm_len = FRM_LEN | (slots << 15) | (slot_len << 18); - if (adata->substream_type == SNDRV_PCM_STREAM_PLAYBACK) { - switch (adata->i2s_instance) { - case I2S_BT_INSTANCE: - reg_val = mmACP_BTTDM_ITER; - frmt_reg = mmACP_BTTDM_TXFRMT; - break; - case I2S_SP_INSTANCE: - default: - reg_val = mmACP_I2STDM_ITER; - frmt_reg = mmACP_I2STDM_TXFRMT; - } - } else { - switch (adata->i2s_instance) { - case I2S_BT_INSTANCE: - reg_val = mmACP_BTTDM_IRER; - frmt_reg = mmACP_BTTDM_RXFRMT; - break; - case I2S_SP_INSTANCE: - default: - reg_val = mmACP_I2STDM_IRER; - frmt_reg = mmACP_I2STDM_RXFRMT; - } - } - val = rv_readl(adata->acp3x_base + reg_val); - rv_writel(val | 0x2, adata->acp3x_base + reg_val); - rv_writel(frm_len, adata->acp3x_base + frmt_reg); adata->tdm_fmt = frm_len; return 0; } @@ -105,12 +76,14 @@ static int acp3x_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_soc_pcm_runtime *prtd; struct snd_soc_card *card; struct acp3x_platform_info *pinfo; + struct i2s_dev_data *adata; u32 val; - u32 reg_val; + u32 reg_val, frmt_reg; prtd = substream->private_data; rtd = substream->runtime->private_data; card = prtd->card; + adata = snd_soc_dai_get_drvdata(dai); pinfo = snd_soc_card_get_drvdata(card); if (pinfo) { if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) @@ -141,21 +114,30 @@ static int acp3x_i2s_hwparams(struct snd_pcm_substream *substream, switch (rtd->i2s_instance) { case I2S_BT_INSTANCE: reg_val = mmACP_BTTDM_ITER; + frmt_reg = mmACP_BTTDM_TXFRMT; break; case I2S_SP_INSTANCE: default: reg_val = mmACP_I2STDM_ITER; + frmt_reg = mmACP_I2STDM_TXFRMT; } } else { switch (rtd->i2s_instance) { case I2S_BT_INSTANCE: reg_val = mmACP_BTTDM_IRER; + frmt_reg = mmACP_BTTDM_RXFRMT; break; case I2S_SP_INSTANCE: default: reg_val = mmACP_I2STDM_IRER; + frmt_reg = mmACP_I2STDM_RXFRMT; } } + if (adata->tdm_mode) { + val = rv_readl(rtd->acp3x_base + reg_val); + rv_writel(val | 0x2, rtd->acp3x_base + reg_val); + rv_writel(adata->tdm_fmt, rtd->acp3x_base + frmt_reg); + } val = rv_readl(rtd->acp3x_base + reg_val); val = val | (rtd->xfer_resolution << 3); rv_writel(val, rtd->acp3x_base + reg_val); diff --git a/sound/soc/amd/raven/acp3x-pcm-dma.c b/sound/soc/amd/raven/acp3x-pcm-dma.c index d62c0d90c41e..e362f0bc9e46 100644 --- a/sound/soc/amd/raven/acp3x-pcm-dma.c +++ b/sound/soc/amd/raven/acp3x-pcm-dma.c @@ -458,7 +458,8 @@ static int acp3x_resume(struct device *dev) reg_val = mmACP_I2STDM_ITER; frmt_val = mmACP_I2STDM_TXFRMT; } - rv_writel((rtd->xfer_resolution << 3), rtd->acp3x_base + reg_val); + rv_writel((rtd->xfer_resolution << 3), + rtd->acp3x_base + reg_val); } if (adata->capture_stream && adata->capture_stream->runtime) { struct i2s_stream_instance *rtd = @@ -474,7 +475,8 @@ static int acp3x_resume(struct device *dev) reg_val = mmACP_I2STDM_IRER; frmt_val = mmACP_I2STDM_RXFRMT; } - rv_writel((rtd->xfer_resolution << 3), rtd->acp3x_base + reg_val); + rv_writel((rtd->xfer_resolution << 3), + rtd->acp3x_base + reg_val); } if (adata->tdm_mode == TDM_ENABLE) { rv_writel(adata->tdm_fmt, adata->acp3x_base + frmt_val); diff --git a/sound/soc/amd/raven/pci-acp3x.c b/sound/soc/amd/raven/pci-acp3x.c index da60e2ec5535..f25ce50f1a90 100644 --- a/sound/soc/amd/raven/pci-acp3x.c +++ b/sound/soc/amd/raven/pci-acp3x.c @@ -38,8 +38,13 @@ static int acp3x_power_on(void __iomem *acp3x_base) timeout = 0; while (++timeout < 500) { val = rv_readl(acp3x_base + mmACP_PGFSM_STATUS); - if (!val) + if (!val) { + /* Set PME_EN as after ACP power On, + * PME_EN gets cleared + */ + rv_writel(0x1, acp3x_base + mmACP_PME_EN); return 0; + } udelay(1); } return -ETIMEDOUT; diff --git a/sound/soc/atmel/atmel-pcm-dma.c b/sound/soc/atmel/atmel-pcm-dma.c index db67f5ba1e9a..cb03c4f7324c 100644 --- a/sound/soc/atmel/atmel-pcm-dma.c +++ b/sound/soc/atmel/atmel-pcm-dma.c @@ -56,7 +56,7 @@ static void atmel_pcm_dma_irq(u32 ssc_sr, struct snd_soc_pcm_runtime *rtd = substream->private_data; struct atmel_pcm_dma_params *prtd; - prtd = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + prtd = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); if (ssc_sr & prtd->mask->ssc_error) { if (snd_pcm_running(substream)) @@ -83,7 +83,7 @@ static int atmel_pcm_configure_dma(struct snd_pcm_substream *substream, struct ssc_device *ssc; int ret; - prtd = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + prtd = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); ssc = prtd->ssc; ret = snd_hwparams_to_dma_slave_config(substream, params, slave_config); diff --git a/sound/soc/atmel/atmel-pcm-pdc.c b/sound/soc/atmel/atmel-pcm-pdc.c index 59c1331a6984..a8daebcbf6c8 100644 --- a/sound/soc/atmel/atmel-pcm-pdc.c +++ b/sound/soc/atmel/atmel-pcm-pdc.c @@ -213,7 +213,7 @@ static int atmel_pcm_hw_params(struct snd_soc_component *component, snd_pcm_set_runtime_buffer(substream, &substream->dma_buffer); runtime->dma_bytes = params_buffer_bytes(params); - prtd->params = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + prtd->params = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); prtd->params->dma_intr_handler = atmel_pcm_dma_irq; prtd->dma_buffer = runtime->dma_addr; diff --git a/sound/soc/atmel/atmel_wm8904.c b/sound/soc/atmel/atmel_wm8904.c index 776b27d3686e..148c943cb538 100644 --- a/sound/soc/atmel/atmel_wm8904.c +++ b/sound/soc/atmel/atmel_wm8904.c @@ -27,7 +27,7 @@ static int atmel_asoc_wm8904_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_pll(codec_dai, WM8904_FLL_MCLK, WM8904_FLL_MCLK, diff --git a/sound/soc/atmel/mchp-i2s-mcc.c b/sound/soc/atmel/mchp-i2s-mcc.c index befc2a3a05b0..3cb63886195f 100644 --- a/sound/soc/atmel/mchp-i2s-mcc.c +++ b/sound/soc/atmel/mchp-i2s-mcc.c @@ -239,10 +239,10 @@ struct mchp_i2s_mcc_dev { unsigned int frame_length; int tdm_slots; int channels; - int gclk_use:1; - int gclk_running:1; - int tx_rdy:1; - int rx_rdy:1; + unsigned int gclk_use:1; + unsigned int gclk_running:1; + unsigned int tx_rdy:1; + unsigned int rx_rdy:1; }; static irqreturn_t mchp_i2s_mcc_interrupt(int irq, void *dev_id) diff --git a/sound/soc/atmel/mikroe-proto.c b/sound/soc/atmel/mikroe-proto.c index aa6d0d78566f..f9a85fd01b79 100644 --- a/sound/soc/atmel/mikroe-proto.c +++ b/sound/soc/atmel/mikroe-proto.c @@ -21,7 +21,7 @@ static int snd_proto_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_card *card = rtd->card; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); /* Set proto sysclk */ int ret = snd_soc_dai_set_sysclk(codec_dai, WM8731_SYSCLK_XTAL, diff --git a/sound/soc/atmel/sam9g20_wm8731.c b/sound/soc/atmel/sam9g20_wm8731.c index b1bef2bf142d..ed1f69b57024 100644 --- a/sound/soc/atmel/sam9g20_wm8731.c +++ b/sound/soc/atmel/sam9g20_wm8731.c @@ -96,7 +96,7 @@ static const struct snd_soc_dapm_route intercon[] = { */ static int at91sam9g20ek_wm8731_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct device *dev = rtd->dev; int ret; diff --git a/sound/soc/atmel/sam9x5_wm8731.c b/sound/soc/atmel/sam9x5_wm8731.c index 7822425d5e61..9fbc3c1113cc 100644 --- a/sound/soc/atmel/sam9x5_wm8731.c +++ b/sound/soc/atmel/sam9x5_wm8731.c @@ -40,7 +40,7 @@ struct sam9x5_drvdata { */ static int sam9x5_wm8731_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct device *dev = rtd->dev; int ret; diff --git a/sound/soc/au1x/db1200.c b/sound/soc/au1x/db1200.c index d6b692fff29a..d649037bda9b 100644 --- a/sound/soc/au1x/db1200.c +++ b/sound/soc/au1x/db1200.c @@ -95,7 +95,7 @@ static struct snd_soc_card db1550_ac97_machine = { static int db1200_i2s_startup(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); /* WM8731 has its own 12MHz crystal */ snd_soc_dai_set_sysclk(codec_dai, WM8731_SYSCLK_XTAL, diff --git a/sound/soc/au1x/dbdma2.c b/sound/soc/au1x/dbdma2.c index 8f855644c6b4..e82bbf2d1eea 100644 --- a/sound/soc/au1x/dbdma2.c +++ b/sound/soc/au1x/dbdma2.c @@ -281,7 +281,7 @@ static int au1xpsc_pcm_open(struct snd_soc_component *component, struct snd_soc_pcm_runtime *rtd = substream->private_data; int stype = substream->stream, *dmaids; - dmaids = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + dmaids = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); if (!dmaids) return -ENODEV; /* whoa, has ordering changed? */ diff --git a/sound/soc/au1x/dma.c b/sound/soc/au1x/dma.c index c9a038a5e2d3..4e246c7e78f2 100644 --- a/sound/soc/au1x/dma.c +++ b/sound/soc/au1x/dma.c @@ -195,7 +195,7 @@ static int alchemy_pcm_open(struct snd_soc_component *component, int *dmaids, s = substream->stream; char *name; - dmaids = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + dmaids = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); if (!dmaids) return -ENODEV; /* whoa, has ordering changed? */ diff --git a/sound/soc/au1x/psc-ac97.c b/sound/soc/au1x/psc-ac97.c index 0227993c5da8..05eb36991f14 100644 --- a/sound/soc/au1x/psc-ac97.c +++ b/sound/soc/au1x/psc-ac97.c @@ -58,7 +58,7 @@ static struct au1xpsc_audio_data *au1xpsc_ac97_workdata; static inline struct au1xpsc_audio_data *ac97_to_pscdata(struct snd_ac97 *x) { struct snd_soc_card *c = x->bus->card->private_data; - return snd_soc_dai_get_drvdata(c->rtd->cpu_dai); + return snd_soc_dai_get_drvdata(c->asoc_rtd_to_cpu(rtd, 0)); } #else diff --git a/sound/soc/bcm/Kconfig b/sound/soc/bcm/Kconfig index 0037e96aa228..4218057b0874 100644 --- a/sound/soc/bcm/Kconfig +++ b/sound/soc/bcm/Kconfig @@ -17,3 +17,12 @@ config SND_SOC_CYGNUS Cygnus chips (bcm958300, bcm958305, bcm911360) If you don't know what to do here, say N. + +config SND_BCM63XX_I2S_WHISTLER + tristate "SoC Audio support for the Broadcom BCM63XX I2S module" + select REGMAP_MMIO + help + Say Y if you want to add support for ASoC audio on Broadcom + DSL/PON chips (bcm63158, bcm63178) + + If you don't know what to do here, say N diff --git a/sound/soc/bcm/Makefile b/sound/soc/bcm/Makefile index b81fa421ec27..7c2d7899603b 100644 --- a/sound/soc/bcm/Makefile +++ b/sound/soc/bcm/Makefile @@ -9,3 +9,7 @@ snd-soc-cygnus-objs := cygnus-pcm.o cygnus-ssp.o obj-$(CONFIG_SND_SOC_CYGNUS) += snd-soc-cygnus.o +# BCM63XX Platform Support +snd-soc-63xx-objs := bcm63xx-i2s-whistler.o bcm63xx-pcm-whistler.o + +obj-$(CONFIG_SND_BCM63XX_I2S_WHISTLER) += snd-soc-63xx.o \ No newline at end of file diff --git a/sound/soc/bcm/bcm63xx-i2s-whistler.c b/sound/soc/bcm/bcm63xx-i2s-whistler.c new file mode 100644 index 000000000000..246a57ac6679 --- /dev/null +++ b/sound/soc/bcm/bcm63xx-i2s-whistler.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// linux/sound/bcm/bcm63xx-i2s-whistler.c +// BCM63xx whistler i2s driver +// Copyright (c) 2020 Broadcom Corporation +// Author: Kevin-Ke Li + +#include +#include +#include +#include +#include +#include +#include +#include "bcm63xx-i2s.h" + +#define DRV_NAME "brcm-i2s" + +static bool brcm_i2s_wr_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case I2S_TX_CFG ... I2S_TX_DESC_IFF_LEN: + case I2S_TX_CFG_2 ... I2S_RX_DESC_IFF_LEN: + case I2S_RX_CFG_2 ... I2S_REG_MAX: + return true; + default: + return false; + } +} + +static bool brcm_i2s_rd_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case I2S_TX_CFG ... I2S_REG_MAX: + return true; + default: + return false; + } +} + +static bool brcm_i2s_volatile_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case I2S_TX_CFG: + case I2S_TX_IRQ_CTL: + case I2S_TX_DESC_IFF_ADDR: + case I2S_TX_DESC_IFF_LEN: + case I2S_TX_DESC_OFF_ADDR: + case I2S_TX_DESC_OFF_LEN: + case I2S_TX_CFG_2: + case I2S_RX_CFG: + case I2S_RX_IRQ_CTL: + case I2S_RX_DESC_OFF_ADDR: + case I2S_RX_DESC_OFF_LEN: + case I2S_RX_DESC_IFF_LEN: + case I2S_RX_DESC_IFF_ADDR: + case I2S_RX_CFG_2: + return true; + default: + return false; + } +} + +static const struct regmap_config brcm_i2s_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = I2S_REG_MAX, + .writeable_reg = brcm_i2s_wr_reg, + .readable_reg = brcm_i2s_rd_reg, + .volatile_reg = brcm_i2s_volatile_reg, + .cache_type = REGCACHE_FLAT, +}; + +static int bcm63xx_i2s_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + int ret = 0; + struct bcm_i2s_priv *i2s_priv = snd_soc_dai_get_drvdata(dai); + + ret = clk_set_rate(i2s_priv->i2s_clk, params_rate(params)); + if (ret < 0) + dev_err(i2s_priv->dev, + "Can't set sample rate, err: %d\n", ret); + + return ret; +} + +static int bcm63xx_i2s_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + unsigned int slavemode; + struct bcm_i2s_priv *i2s_priv = snd_soc_dai_get_drvdata(dai); + struct regmap *regmap_i2s = i2s_priv->regmap_i2s; + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + regmap_update_bits(regmap_i2s, I2S_TX_CFG, + I2S_TX_OUT_R | I2S_TX_DATA_ALIGNMENT | + I2S_TX_DATA_ENABLE | I2S_TX_CLOCK_ENABLE, + I2S_TX_OUT_R | I2S_TX_DATA_ALIGNMENT | + I2S_TX_DATA_ENABLE | I2S_TX_CLOCK_ENABLE); + regmap_write(regmap_i2s, I2S_TX_IRQ_CTL, 0); + regmap_write(regmap_i2s, I2S_TX_IRQ_IFF_THLD, 0); + regmap_write(regmap_i2s, I2S_TX_IRQ_OFF_THLD, 1); + + /* TX and RX block each have an independent bit to indicate + * if it is generating the clock for the I2S bus. The bus + * clocks need to be generated from either the TX or RX block, + * but not both + */ + regmap_read(regmap_i2s, I2S_RX_CFG_2, &slavemode); + if (slavemode & I2S_RX_SLAVE_MODE_MASK) + regmap_update_bits(regmap_i2s, I2S_TX_CFG_2, + I2S_TX_SLAVE_MODE_MASK, + I2S_TX_MASTER_MODE); + else + regmap_update_bits(regmap_i2s, I2S_TX_CFG_2, + I2S_TX_SLAVE_MODE_MASK, + I2S_TX_SLAVE_MODE); + } else { + regmap_update_bits(regmap_i2s, I2S_RX_CFG, + I2S_RX_IN_R | I2S_RX_DATA_ALIGNMENT | + I2S_RX_CLOCK_ENABLE, + I2S_RX_IN_R | I2S_RX_DATA_ALIGNMENT | + I2S_RX_CLOCK_ENABLE); + regmap_write(regmap_i2s, I2S_RX_IRQ_CTL, 0); + regmap_write(regmap_i2s, I2S_RX_IRQ_IFF_THLD, 0); + regmap_write(regmap_i2s, I2S_RX_IRQ_OFF_THLD, 1); + + regmap_read(regmap_i2s, I2S_TX_CFG_2, &slavemode); + if (slavemode & I2S_TX_SLAVE_MODE_MASK) + regmap_update_bits(regmap_i2s, I2S_RX_CFG_2, + I2S_RX_SLAVE_MODE_MASK, 0); + else + regmap_update_bits(regmap_i2s, I2S_RX_CFG_2, + I2S_RX_SLAVE_MODE_MASK, + I2S_RX_SLAVE_MODE); + } + return 0; +} + +static void bcm63xx_i2s_shutdown(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + unsigned int enabled, slavemode; + struct bcm_i2s_priv *i2s_priv = snd_soc_dai_get_drvdata(dai); + struct regmap *regmap_i2s = i2s_priv->regmap_i2s; + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + regmap_update_bits(regmap_i2s, I2S_TX_CFG, + I2S_TX_OUT_R | I2S_TX_DATA_ALIGNMENT | + I2S_TX_DATA_ENABLE | I2S_TX_CLOCK_ENABLE, 0); + regmap_write(regmap_i2s, I2S_TX_IRQ_CTL, 1); + regmap_write(regmap_i2s, I2S_TX_IRQ_IFF_THLD, 4); + regmap_write(regmap_i2s, I2S_TX_IRQ_OFF_THLD, 4); + + regmap_read(regmap_i2s, I2S_TX_CFG_2, &slavemode); + slavemode = slavemode & I2S_TX_SLAVE_MODE_MASK; + if (!slavemode) { + regmap_read(regmap_i2s, I2S_RX_CFG, &enabled); + enabled = enabled & I2S_RX_ENABLE_MASK; + if (enabled) + regmap_update_bits(regmap_i2s, I2S_RX_CFG_2, + I2S_RX_SLAVE_MODE_MASK, + I2S_RX_MASTER_MODE); + } + regmap_update_bits(regmap_i2s, I2S_TX_CFG_2, + I2S_TX_SLAVE_MODE_MASK, + I2S_TX_SLAVE_MODE); + } else { + regmap_update_bits(regmap_i2s, I2S_RX_CFG, + I2S_RX_IN_R | I2S_RX_DATA_ALIGNMENT | + I2S_RX_CLOCK_ENABLE, 0); + regmap_write(regmap_i2s, I2S_RX_IRQ_CTL, 1); + regmap_write(regmap_i2s, I2S_RX_IRQ_IFF_THLD, 4); + regmap_write(regmap_i2s, I2S_RX_IRQ_OFF_THLD, 4); + + regmap_read(regmap_i2s, I2S_RX_CFG_2, &slavemode); + slavemode = slavemode & I2S_RX_SLAVE_MODE_MASK; + if (!slavemode) { + regmap_read(regmap_i2s, I2S_TX_CFG, &enabled); + enabled = enabled & I2S_TX_ENABLE_MASK; + if (enabled) + regmap_update_bits(regmap_i2s, I2S_TX_CFG_2, + I2S_TX_SLAVE_MODE_MASK, + I2S_TX_MASTER_MODE); + } + + regmap_update_bits(regmap_i2s, I2S_RX_CFG_2, + I2S_RX_SLAVE_MODE_MASK, I2S_RX_SLAVE_MODE); + } +} + +static const struct snd_soc_dai_ops bcm63xx_i2s_dai_ops = { + .startup = bcm63xx_i2s_startup, + .shutdown = bcm63xx_i2s_shutdown, + .hw_params = bcm63xx_i2s_hw_params, +}; + +static struct snd_soc_dai_driver bcm63xx_i2s_dai = { + .name = DRV_NAME, + .playback = { + .channels_min = 2, + .channels_max = 2, + .rates = SNDRV_PCM_RATE_8000_192000, + .formats = SNDRV_PCM_FMTBIT_S32_LE, + }, + .capture = { + .channels_min = 2, + .channels_max = 2, + .rates = SNDRV_PCM_RATE_8000_192000, + .formats = SNDRV_PCM_FMTBIT_S32_LE, + }, + .ops = &bcm63xx_i2s_dai_ops, + .symmetric_rates = 1, + .symmetric_channels = 1, +}; + +static const struct snd_soc_component_driver bcm63xx_i2s_component = { + .name = "bcm63xx", +}; + +static int bcm63xx_i2s_dev_probe(struct platform_device *pdev) +{ + int ret = 0; + void __iomem *regs; + struct resource *r_mem, *region; + struct bcm_i2s_priv *i2s_priv; + struct regmap *regmap_i2s; + struct clk *i2s_clk; + + i2s_priv = devm_kzalloc(&pdev->dev, sizeof(*i2s_priv), GFP_KERNEL); + if (!i2s_priv) + return -ENOMEM; + + i2s_clk = devm_clk_get(&pdev->dev, "i2sclk"); + if (IS_ERR(i2s_clk)) { + dev_err(&pdev->dev, "%s: cannot get a brcm clock: %ld\n", + __func__, PTR_ERR(i2s_clk)); + return PTR_ERR(i2s_clk); + } + + r_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!r_mem) { + dev_err(&pdev->dev, "Unable to get register resource.\n"); + return -ENODEV; + } + + region = devm_request_mem_region(&pdev->dev, r_mem->start, + resource_size(r_mem), DRV_NAME); + if (!region) { + dev_err(&pdev->dev, "Memory region already claimed\n"); + return -EBUSY; + } + + regs = devm_ioremap_resource(&pdev->dev, r_mem); + if (IS_ERR(regs)) { + ret = PTR_ERR(regs); + return ret; + } + + regmap_i2s = devm_regmap_init_mmio(&pdev->dev, + regs, &brcm_i2s_regmap_config); + if (IS_ERR(regmap_i2s)) + return PTR_ERR(regmap_i2s); + + regmap_update_bits(regmap_i2s, I2S_MISC_CFG, + I2S_PAD_LVL_LOOP_DIS_MASK, + I2S_PAD_LVL_LOOP_DIS_ENABLE); + + ret = devm_snd_soc_register_component(&pdev->dev, + &bcm63xx_i2s_component, + &bcm63xx_i2s_dai, 1); + if (ret) { + dev_err(&pdev->dev, "failed to register the dai\n"); + return ret; + } + + i2s_priv->dev = &pdev->dev; + i2s_priv->i2s_clk = i2s_clk; + i2s_priv->regmap_i2s = regmap_i2s; + dev_set_drvdata(&pdev->dev, i2s_priv); + + ret = bcm63xx_soc_platform_probe(pdev, i2s_priv); + if (ret) + dev_err(&pdev->dev, "failed to register the pcm\n"); + + return ret; +} + +static int bcm63xx_i2s_dev_remove(struct platform_device *pdev) +{ + bcm63xx_soc_platform_remove(pdev); + return 0; +} + +#ifdef CONFIG_OF +static const struct of_device_id snd_soc_bcm_audio_match[] = { + {.compatible = "brcm,bcm63xx-i2s"}, + { } +}; +#endif + +static struct platform_driver bcm63xx_i2s_driver = { + .driver = { + .name = DRV_NAME, + .of_match_table = of_match_ptr(snd_soc_bcm_audio_match), + }, + .probe = bcm63xx_i2s_dev_probe, + .remove = bcm63xx_i2s_dev_remove, +}; + +module_platform_driver(bcm63xx_i2s_driver); + +MODULE_AUTHOR("Kevin,Li "); +MODULE_DESCRIPTION("Broadcom DSL XPON ASOC I2S Interface"); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/bcm/bcm63xx-i2s.h b/sound/soc/bcm/bcm63xx-i2s.h new file mode 100644 index 000000000000..edc328ba53d3 --- /dev/null +++ b/sound/soc/bcm/bcm63xx-i2s.h @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// linux/sound/soc/bcm/bcm63xx-i2s.h +// Copyright (c) 2020 Broadcom Corporation +// Author: Kevin-Ke Li + +#ifndef __BCM63XX_I2S_H +#define __BCM63XX_I2S_H + +#define I2S_DESC_FIFO_DEPTH 8 +#define I2S_MISC_CFG (0x003C) +#define I2S_PAD_LVL_LOOP_DIS_MASK (1 << 2) +#define I2S_PAD_LVL_LOOP_DIS_ENABLE I2S_PAD_LVL_LOOP_DIS_MASK + +#define I2S_TX_ENABLE_MASK (1 << 31) +#define I2S_TX_ENABLE I2S_TX_ENABLE_MASK +#define I2S_TX_OUT_R (1 << 19) +#define I2S_TX_DATA_ALIGNMENT (1 << 2) +#define I2S_TX_DATA_ENABLE (1 << 1) +#define I2S_TX_CLOCK_ENABLE (1 << 0) + +#define I2S_TX_DESC_OFF_LEVEL_SHIFT 12 +#define I2S_TX_DESC_OFF_LEVEL_MASK (0x0F << I2S_TX_DESC_OFF_LEVEL_SHIFT) +#define I2S_TX_DESC_IFF_LEVEL_SHIFT 8 +#define I2S_TX_DESC_IFF_LEVEL_MASK (0x0F << I2S_TX_DESC_IFF_LEVEL_SHIFT) +#define I2S_TX_DESC_OFF_INTR_EN_MSK (1 << 1) +#define I2S_TX_DESC_OFF_INTR_EN I2S_TX_DESC_OFF_INTR_EN_MSK + +#define I2S_TX_CFG (0x0000) +#define I2S_TX_IRQ_CTL (0x0004) +#define I2S_TX_IRQ_EN (0x0008) +#define I2S_TX_IRQ_IFF_THLD (0x000c) +#define I2S_TX_IRQ_OFF_THLD (0x0010) +#define I2S_TX_DESC_IFF_ADDR (0x0014) +#define I2S_TX_DESC_IFF_LEN (0x0018) +#define I2S_TX_DESC_OFF_ADDR (0x001C) +#define I2S_TX_DESC_OFF_LEN (0x0020) +#define I2S_TX_CFG_2 (0x0024) +#define I2S_TX_SLAVE_MODE_SHIFT 13 +#define I2S_TX_SLAVE_MODE_MASK (1 << I2S_TX_SLAVE_MODE_SHIFT) +#define I2S_TX_SLAVE_MODE I2S_TX_SLAVE_MODE_MASK +#define I2S_TX_MASTER_MODE 0 +#define I2S_TX_INTR_MASK 0x0F + +#define I2S_RX_ENABLE_MASK (1 << 31) +#define I2S_RX_ENABLE I2S_RX_ENABLE_MASK +#define I2S_RX_IN_R (1 << 19) +#define I2S_RX_DATA_ALIGNMENT (1 << 2) +#define I2S_RX_CLOCK_ENABLE (1 << 0) + +#define I2S_RX_DESC_OFF_LEVEL_SHIFT 12 +#define I2S_RX_DESC_OFF_LEVEL_MASK (0x0F << I2S_RX_DESC_OFF_LEVEL_SHIFT) +#define I2S_RX_DESC_IFF_LEVEL_SHIFT 8 +#define I2S_RX_DESC_IFF_LEVEL_MASK (0x0F << I2S_RX_DESC_IFF_LEVEL_SHIFT) +#define I2S_RX_DESC_OFF_INTR_EN_MSK (1 << 1) +#define I2S_RX_DESC_OFF_INTR_EN I2S_RX_DESC_OFF_INTR_EN_MSK + +#define I2S_RX_CFG (0x0040) /* 20c0 */ +#define I2S_RX_IRQ_CTL (0x0044) +#define I2S_RX_IRQ_EN (0x0048) +#define I2S_RX_IRQ_IFF_THLD (0x004C) +#define I2S_RX_IRQ_OFF_THLD (0x0050) +#define I2S_RX_DESC_IFF_ADDR (0x0054) +#define I2S_RX_DESC_IFF_LEN (0x0058) +#define I2S_RX_DESC_OFF_ADDR (0x005C) +#define I2S_RX_DESC_OFF_LEN (0x0060) +#define I2S_RX_CFG_2 (0x0064) +#define I2S_RX_SLAVE_MODE_SHIFT 13 +#define I2S_RX_SLAVE_MODE_MASK (1 << I2S_RX_SLAVE_MODE_SHIFT) +#define I2S_RX_SLAVE_MODE I2S_RX_SLAVE_MODE_MASK +#define I2S_RX_MASTER_MODE 0 +#define I2S_RX_INTR_MASK 0x0F + +#define I2S_REG_MAX 0x007C + +struct bcm_i2s_priv { + struct device *dev; + struct resource *r_irq; + struct regmap *regmap_i2s; + struct clk *i2s_clk; + struct snd_pcm_substream *play_substream; + struct snd_pcm_substream *capture_substream; + struct i2s_dma_desc *play_dma_desc; + struct i2s_dma_desc *capture_dma_desc; +}; + +extern int bcm63xx_soc_platform_probe(struct platform_device *pdev, + struct bcm_i2s_priv *i2s_priv); +extern int bcm63xx_soc_platform_remove(struct platform_device *pdev); + +#endif diff --git a/sound/soc/bcm/bcm63xx-pcm-whistler.c b/sound/soc/bcm/bcm63xx-pcm-whistler.c new file mode 100644 index 000000000000..e46c390683e7 --- /dev/null +++ b/sound/soc/bcm/bcm63xx-pcm-whistler.c @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// linux/sound/bcm/bcm63xx-pcm-whistler.c +// BCM63xx whistler pcm interface +// Copyright (c) 2020 Broadcom Corporation +// Author: Kevin-Ke Li + +#include +#include +#include +#include +#include +#include +#include +#include "bcm63xx-i2s.h" + + +struct i2s_dma_desc { + unsigned char *dma_area; + dma_addr_t dma_addr; + unsigned int dma_len; +}; + +struct bcm63xx_runtime_data { + int dma_len; + dma_addr_t dma_addr; + dma_addr_t dma_addr_next; +}; + +static const struct snd_pcm_hardware bcm63xx_pcm_hardware = { + .info = SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED | + SNDRV_PCM_INFO_PAUSE | + SNDRV_PCM_INFO_RESUME, + .formats = SNDRV_PCM_FMTBIT_S32_LE, /* support S32 only */ + .period_bytes_max = 8192 - 32, + .periods_min = 1, + .periods_max = PAGE_SIZE/sizeof(struct i2s_dma_desc), + .buffer_bytes_max = 128 * 1024, + .fifo_size = 32, +}; + +static int bcm63xx_pcm_hw_params(struct snd_soc_component *component, + struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct i2s_dma_desc *dma_desc; + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_pcm_runtime *runtime = substream->runtime; + + snd_pcm_set_runtime_buffer(substream, &substream->dma_buffer); + runtime->dma_bytes = params_buffer_bytes(params); + + dma_desc = kzalloc(sizeof(*dma_desc), GFP_NOWAIT); + if (!dma_desc) + return -ENOMEM; + + snd_soc_dai_set_dma_data(asoc_rtd_to_cpu(rtd, 0), substream, dma_desc); + + return 0; +} + +static int bcm63xx_pcm_hw_free(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + struct i2s_dma_desc *dma_desc; + struct snd_soc_pcm_runtime *rtd = substream->private_data; + + dma_desc = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); + kfree(dma_desc); + snd_pcm_set_runtime_buffer(substream, NULL); + + return 0; +} + +static int bcm63xx_pcm_trigger(struct snd_soc_component *component, + struct snd_pcm_substream *substream, int cmd) +{ + int ret = 0; + struct snd_soc_pcm_runtime *rtd; + struct bcm_i2s_priv *i2s_priv; + struct regmap *regmap_i2s; + + rtd = substream->private_data; + i2s_priv = dev_get_drvdata(asoc_rtd_to_cpu(rtd, 0)->dev); + regmap_i2s = i2s_priv->regmap_i2s; + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + regmap_update_bits(regmap_i2s, + I2S_TX_IRQ_EN, + I2S_TX_DESC_OFF_INTR_EN, + I2S_TX_DESC_OFF_INTR_EN); + regmap_update_bits(regmap_i2s, + I2S_TX_CFG, + I2S_TX_ENABLE_MASK, + I2S_TX_ENABLE); + break; + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + regmap_write(regmap_i2s, + I2S_TX_IRQ_EN, + 0); + regmap_update_bits(regmap_i2s, + I2S_TX_CFG, + I2S_TX_ENABLE_MASK, + 0); + break; + default: + ret = -EINVAL; + } + } else { + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + regmap_update_bits(regmap_i2s, + I2S_RX_IRQ_EN, + I2S_RX_DESC_OFF_INTR_EN_MSK, + I2S_RX_DESC_OFF_INTR_EN); + regmap_update_bits(regmap_i2s, + I2S_RX_CFG, + I2S_RX_ENABLE_MASK, + I2S_RX_ENABLE); + break; + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + regmap_update_bits(regmap_i2s, + I2S_RX_IRQ_EN, + I2S_RX_DESC_OFF_INTR_EN_MSK, + 0); + regmap_update_bits(regmap_i2s, + I2S_RX_CFG, + I2S_RX_ENABLE_MASK, + 0); + break; + default: + ret = -EINVAL; + } + } + return ret; +} + +static int bcm63xx_pcm_prepare(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + struct i2s_dma_desc *dma_desc; + struct regmap *regmap_i2s; + struct bcm_i2s_priv *i2s_priv; + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_pcm_runtime *runtime = substream->runtime; + uint32_t regaddr_desclen, regaddr_descaddr; + + dma_desc = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); + dma_desc->dma_len = snd_pcm_lib_period_bytes(substream); + dma_desc->dma_addr = runtime->dma_addr; + dma_desc->dma_area = runtime->dma_area; + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + regaddr_desclen = I2S_TX_DESC_IFF_LEN; + regaddr_descaddr = I2S_TX_DESC_IFF_ADDR; + } else { + regaddr_desclen = I2S_RX_DESC_IFF_LEN; + regaddr_descaddr = I2S_RX_DESC_IFF_ADDR; + } + + i2s_priv = dev_get_drvdata(asoc_rtd_to_cpu(rtd, 0)->dev); + regmap_i2s = i2s_priv->regmap_i2s; + + regmap_write(regmap_i2s, regaddr_desclen, dma_desc->dma_len); + regmap_write(regmap_i2s, regaddr_descaddr, dma_desc->dma_addr); + + return 0; +} + +static snd_pcm_uframes_t +bcm63xx_pcm_pointer(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + snd_pcm_uframes_t x; + struct bcm63xx_runtime_data *prtd = substream->runtime->private_data; + + if ((void *)prtd->dma_addr_next == NULL) + prtd->dma_addr_next = substream->runtime->dma_addr; + + x = bytes_to_frames(substream->runtime, + prtd->dma_addr_next - substream->runtime->dma_addr); + + return x == substream->runtime->buffer_size ? 0 : x; +} + +static int bcm63xx_pcm_mmap(struct snd_soc_component *component, + struct snd_pcm_substream *substream, + struct vm_area_struct *vma) +{ + struct snd_pcm_runtime *runtime = substream->runtime; + + return dma_mmap_wc(substream->pcm->card->dev, vma, + runtime->dma_area, + runtime->dma_addr, + runtime->dma_bytes); + +} + +static int bcm63xx_pcm_open(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + int ret = 0; + struct snd_pcm_runtime *runtime = substream->runtime; + struct bcm63xx_runtime_data *prtd; + + runtime->hw = bcm63xx_pcm_hardware; + ret = snd_pcm_hw_constraint_step(runtime, 0, + SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 32); + if (ret) + goto out; + + ret = snd_pcm_hw_constraint_step(runtime, 0, + SNDRV_PCM_HW_PARAM_BUFFER_BYTES, 32); + if (ret) + goto out; + + ret = snd_pcm_hw_constraint_integer(runtime, + SNDRV_PCM_HW_PARAM_PERIODS); + if (ret < 0) + goto out; + + ret = -ENOMEM; + prtd = kzalloc(sizeof(*prtd), GFP_KERNEL); + if (!prtd) + goto out; + + runtime->private_data = prtd; + return 0; +out: + return ret; +} + +static int bcm63xx_pcm_close(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + struct snd_pcm_runtime *runtime = substream->runtime; + struct bcm63xx_runtime_data *prtd = runtime->private_data; + + kfree(prtd); + return 0; +} + +static irqreturn_t i2s_dma_isr(int irq, void *bcm_i2s_priv) +{ + unsigned int availdepth, ifflevel, offlevel, int_status, val_1, val_2; + struct bcm63xx_runtime_data *prtd; + struct snd_pcm_substream *substream; + struct snd_pcm_runtime *runtime; + struct regmap *regmap_i2s; + struct i2s_dma_desc *dma_desc; + struct snd_soc_pcm_runtime *rtd; + struct bcm_i2s_priv *i2s_priv; + + i2s_priv = (struct bcm_i2s_priv *)bcm_i2s_priv; + regmap_i2s = i2s_priv->regmap_i2s; + + /* rx */ + regmap_read(regmap_i2s, I2S_RX_IRQ_CTL, &int_status); + + if (int_status & I2S_RX_DESC_OFF_INTR_EN_MSK) { + substream = i2s_priv->capture_substream; + runtime = substream->runtime; + rtd = substream->private_data; + prtd = runtime->private_data; + dma_desc = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); + + offlevel = (int_status & I2S_RX_DESC_OFF_LEVEL_MASK) >> + I2S_RX_DESC_OFF_LEVEL_SHIFT; + while (offlevel) { + regmap_read(regmap_i2s, I2S_RX_DESC_OFF_ADDR, &val_1); + regmap_read(regmap_i2s, I2S_RX_DESC_OFF_LEN, &val_2); + offlevel--; + } + prtd->dma_addr_next = val_1 + val_2; + ifflevel = (int_status & I2S_RX_DESC_IFF_LEVEL_MASK) >> + I2S_RX_DESC_IFF_LEVEL_SHIFT; + + availdepth = I2S_DESC_FIFO_DEPTH - ifflevel; + while (availdepth) { + dma_desc->dma_addr += + snd_pcm_lib_period_bytes(substream); + dma_desc->dma_area += + snd_pcm_lib_period_bytes(substream); + if (dma_desc->dma_addr - runtime->dma_addr >= + runtime->dma_bytes) { + dma_desc->dma_addr = runtime->dma_addr; + dma_desc->dma_area = runtime->dma_area; + } + + prtd->dma_addr = dma_desc->dma_addr; + regmap_write(regmap_i2s, I2S_RX_DESC_IFF_LEN, + snd_pcm_lib_period_bytes(substream)); + regmap_write(regmap_i2s, I2S_RX_DESC_IFF_ADDR, + dma_desc->dma_addr); + availdepth--; + } + + snd_pcm_period_elapsed(substream); + + /* Clear interrupt by writing 0 */ + regmap_update_bits(regmap_i2s, I2S_RX_IRQ_CTL, + I2S_RX_INTR_MASK, 0); + } + + /* tx */ + regmap_read(regmap_i2s, I2S_TX_IRQ_CTL, &int_status); + + if (int_status & I2S_TX_DESC_OFF_INTR_EN_MSK) { + substream = i2s_priv->play_substream; + runtime = substream->runtime; + rtd = substream->private_data; + prtd = runtime->private_data; + dma_desc = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); + + offlevel = (int_status & I2S_TX_DESC_OFF_LEVEL_MASK) >> + I2S_TX_DESC_OFF_LEVEL_SHIFT; + while (offlevel) { + regmap_read(regmap_i2s, I2S_TX_DESC_OFF_ADDR, &val_1); + regmap_read(regmap_i2s, I2S_TX_DESC_OFF_LEN, &val_2); + prtd->dma_addr_next = val_1 + val_2; + offlevel--; + } + + ifflevel = (int_status & I2S_TX_DESC_IFF_LEVEL_MASK) >> + I2S_TX_DESC_IFF_LEVEL_SHIFT; + availdepth = I2S_DESC_FIFO_DEPTH - ifflevel; + + while (availdepth) { + dma_desc->dma_addr += + snd_pcm_lib_period_bytes(substream); + dma_desc->dma_area += + snd_pcm_lib_period_bytes(substream); + + if (dma_desc->dma_addr - runtime->dma_addr >= + runtime->dma_bytes) { + dma_desc->dma_addr = runtime->dma_addr; + dma_desc->dma_area = runtime->dma_area; + } + + prtd->dma_addr = dma_desc->dma_addr; + regmap_write(regmap_i2s, I2S_TX_DESC_IFF_LEN, + snd_pcm_lib_period_bytes(substream)); + regmap_write(regmap_i2s, I2S_TX_DESC_IFF_ADDR, + dma_desc->dma_addr); + availdepth--; + } + + snd_pcm_period_elapsed(substream); + + /* Clear interrupt by writing 0 */ + regmap_update_bits(regmap_i2s, I2S_TX_IRQ_CTL, + I2S_TX_INTR_MASK, 0); + } + + return IRQ_HANDLED; +} + +static int bcm63xx_pcm_preallocate_dma_buffer(struct snd_pcm *pcm, int stream) +{ + struct snd_pcm_substream *substream = pcm->streams[stream].substream; + struct snd_dma_buffer *buf = &substream->dma_buffer; + size_t size = bcm63xx_pcm_hardware.buffer_bytes_max; + + buf->dev.type = SNDRV_DMA_TYPE_DEV; + buf->dev.dev = pcm->card->dev; + buf->private_data = NULL; + + buf->area = dma_alloc_wc(pcm->card->dev, + size, &buf->addr, + GFP_KERNEL); + if (!buf->area) + return -ENOMEM; + buf->bytes = size; + return 0; +} + +static int bcm63xx_soc_pcm_new(struct snd_soc_component *component, + struct snd_soc_pcm_runtime *rtd) +{ + struct snd_pcm *pcm = rtd->pcm; + struct bcm_i2s_priv *i2s_priv; + int ret; + + i2s_priv = dev_get_drvdata(asoc_rtd_to_cpu(rtd, 0)->dev); + + of_dma_configure(pcm->card->dev, pcm->card->dev->of_node, 1); + + ret = dma_coerce_mask_and_coherent(pcm->card->dev, DMA_BIT_MASK(32)); + if (ret) + goto out; + + if (pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream) { + ret = bcm63xx_pcm_preallocate_dma_buffer(pcm, + SNDRV_PCM_STREAM_PLAYBACK); + if (ret) + goto out; + + i2s_priv->play_substream = + pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + } + + if (pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream) { + ret = bcm63xx_pcm_preallocate_dma_buffer(pcm, + SNDRV_PCM_STREAM_CAPTURE); + if (ret) + goto out; + i2s_priv->capture_substream = + pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream; + } + +out: + return ret; +} + +static void bcm63xx_pcm_free_dma_buffers(struct snd_soc_component *component, + struct snd_pcm *pcm) +{ + int stream; + struct snd_dma_buffer *buf; + struct snd_pcm_substream *substream; + + for (stream = 0; stream < 2; stream++) { + substream = pcm->streams[stream].substream; + if (!substream) + continue; + buf = &substream->dma_buffer; + if (!buf->area) + continue; + dma_free_wc(pcm->card->dev, buf->bytes, + buf->area, buf->addr); + buf->area = NULL; + } +} + +static const struct snd_soc_component_driver bcm63xx_soc_platform = { + .open = bcm63xx_pcm_open, + .close = bcm63xx_pcm_close, + .hw_params = bcm63xx_pcm_hw_params, + .hw_free = bcm63xx_pcm_hw_free, + .prepare = bcm63xx_pcm_prepare, + .trigger = bcm63xx_pcm_trigger, + .pointer = bcm63xx_pcm_pointer, + .mmap = bcm63xx_pcm_mmap, + .pcm_construct = bcm63xx_soc_pcm_new, + .pcm_destruct = bcm63xx_pcm_free_dma_buffers, +}; + +int bcm63xx_soc_platform_probe(struct platform_device *pdev, + struct bcm_i2s_priv *i2s_priv) +{ + int ret; + + i2s_priv->r_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!i2s_priv->r_irq) { + dev_err(&pdev->dev, "Unable to get register irq resource.\n"); + return -ENODEV; + } + + ret = devm_request_irq(&pdev->dev, i2s_priv->r_irq->start, i2s_dma_isr, + i2s_priv->r_irq->flags, "i2s_dma", (void *)i2s_priv); + if (ret) { + dev_err(&pdev->dev, + "i2s_init: failed to request interrupt.ret=%d\n", ret); + return ret; + } + + return devm_snd_soc_register_component(&pdev->dev, + &bcm63xx_soc_platform, NULL, 0); +} + +int bcm63xx_soc_platform_remove(struct platform_device *pdev) +{ + return 0; +} + +MODULE_AUTHOR("Kevin,Li "); +MODULE_DESCRIPTION("Broadcom DSL XPON ASOC PCM Interface"); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/bcm/cygnus-pcm.c b/sound/soc/bcm/cygnus-pcm.c index 3a80c613bc3f..f96d27c8b301 100644 --- a/sound/soc/bcm/cygnus-pcm.c +++ b/sound/soc/bcm/cygnus-pcm.c @@ -209,7 +209,7 @@ static struct cygnus_aio_port *cygnus_dai_get_dma_data( { struct snd_soc_pcm_runtime *soc_runtime = substream->private_data; - return snd_soc_dai_get_dma_data(soc_runtime->cpu_dai, substream); + return snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(soc_runtime, 0), substream); } static void ringbuf_set_initial(void __iomem *audio_io, @@ -359,7 +359,7 @@ static void disable_intr(struct snd_pcm_substream *substream) aio = cygnus_dai_get_dma_data(substream); - dev_dbg(rtd->cpu_dai->dev, "%s on port %d\n", __func__, aio->portnum); + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "%s on port %d\n", __func__, aio->portnum); /* The port number maps to the bit position to be set */ set_mask = BIT(aio->portnum); @@ -590,7 +590,7 @@ static int cygnus_pcm_open(struct snd_soc_component *component, if (!aio) return -ENODEV; - dev_dbg(rtd->cpu_dai->dev, "%s port %d\n", __func__, aio->portnum); + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "%s port %d\n", __func__, aio->portnum); snd_soc_set_runtime_hwparams(substream, &cygnus_pcm_hw); @@ -623,7 +623,7 @@ static int cygnus_pcm_close(struct snd_soc_component *component, aio = cygnus_dai_get_dma_data(substream); - dev_dbg(rtd->cpu_dai->dev, "%s port %d\n", __func__, aio->portnum); + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "%s port %d\n", __func__, aio->portnum); if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) aio->play_stream = NULL; @@ -631,7 +631,7 @@ static int cygnus_pcm_close(struct snd_soc_component *component, aio->capture_stream = NULL; if (!aio->play_stream && !aio->capture_stream) - dev_dbg(rtd->cpu_dai->dev, "freed port %d\n", aio->portnum); + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "freed port %d\n", aio->portnum); return 0; } @@ -645,7 +645,7 @@ static int cygnus_pcm_hw_params(struct snd_soc_component *component, struct cygnus_aio_port *aio; aio = cygnus_dai_get_dma_data(substream); - dev_dbg(rtd->cpu_dai->dev, "%s port %d\n", __func__, aio->portnum); + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "%s port %d\n", __func__, aio->portnum); snd_pcm_set_runtime_buffer(substream, &substream->dma_buffer); runtime->dma_bytes = params_buffer_bytes(params); @@ -660,7 +660,7 @@ static int cygnus_pcm_hw_free(struct snd_soc_component *component, struct cygnus_aio_port *aio; aio = cygnus_dai_get_dma_data(substream); - dev_dbg(rtd->cpu_dai->dev, "%s port %d\n", __func__, aio->portnum); + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "%s port %d\n", __func__, aio->portnum); snd_pcm_set_runtime_buffer(substream, NULL); return 0; @@ -678,12 +678,12 @@ static int cygnus_pcm_prepare(struct snd_soc_component *component, struct ringbuf_regs *p_rbuf = NULL; aio = cygnus_dai_get_dma_data(substream); - dev_dbg(rtd->cpu_dai->dev, "%s port %d\n", __func__, aio->portnum); + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "%s port %d\n", __func__, aio->portnum); bufsize = snd_pcm_lib_buffer_bytes(substream); periodsize = snd_pcm_lib_period_bytes(substream); - dev_dbg(rtd->cpu_dai->dev, "%s (buf_size %lu) (period_size %lu)\n", + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "%s (buf_size %lu) (period_size %lu)\n", __func__, bufsize, periodsize); configure_ringbuf_regs(substream); @@ -745,11 +745,11 @@ static int cygnus_pcm_preallocate_dma_buffer(struct snd_pcm *pcm, int stream) buf->area = dma_alloc_coherent(pcm->card->dev, size, &buf->addr, GFP_KERNEL); - dev_dbg(rtd->cpu_dai->dev, "%s: size 0x%zx @ %pK\n", + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "%s: size 0x%zx @ %pK\n", __func__, size, buf->area); if (!buf->area) { - dev_err(rtd->cpu_dai->dev, "%s: dma_alloc failed\n", __func__); + dev_err(asoc_rtd_to_cpu(rtd, 0)->dev, "%s: dma_alloc failed\n", __func__); return -ENOMEM; } buf->bytes = size; diff --git a/sound/soc/cirrus/edb93xx.c b/sound/soc/cirrus/edb93xx.c index 10961190068e..ccf65f087ea6 100644 --- a/sound/soc/cirrus/edb93xx.c +++ b/sound/soc/cirrus/edb93xx.c @@ -23,8 +23,8 @@ static int edb93xx_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int err; unsigned int mclk_rate; unsigned int rate = params_rate(params); diff --git a/sound/soc/cirrus/snappercl15.c b/sound/soc/cirrus/snappercl15.c index 70c2f3e08d6d..cb133e80b7c3 100644 --- a/sound/soc/cirrus/snappercl15.c +++ b/sound/soc/cirrus/snappercl15.c @@ -23,8 +23,8 @@ static int snappercl15_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int err; err = snd_soc_dai_set_sysclk(codec_dai, 0, CODEC_CLOCK, diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig index ea912439e446..e6a0c5d05fa5 100644 --- a/sound/soc/codecs/Kconfig +++ b/sound/soc/codecs/Kconfig @@ -14,262 +14,264 @@ menu "CODEC drivers" config SND_SOC_ALL_CODECS tristate "Build all ASoC CODEC drivers" depends on COMPILE_TEST - select SND_SOC_88PM860X if MFD_88PM860X - select SND_SOC_L3 - select SND_SOC_AB8500_CODEC if ABX500_CORE - select SND_SOC_AC97_CODEC - select SND_SOC_AD1836 if SPI_MASTER - select SND_SOC_AD193X_SPI if SPI_MASTER - select SND_SOC_AD193X_I2C if I2C - select SND_SOC_AD1980 if SND_SOC_AC97_BUS - select SND_SOC_AD73311 - select SND_SOC_ADAU1373 if I2C - select SND_SOC_ADAU1761_I2C if I2C - select SND_SOC_ADAU1761_SPI if SPI - select SND_SOC_ADAU1781_I2C if I2C - select SND_SOC_ADAU1781_SPI if SPI - select SND_SOC_ADAV801 if SPI_MASTER - select SND_SOC_ADAV803 if I2C - select SND_SOC_ADAU1977_SPI if SPI_MASTER - select SND_SOC_ADAU1977_I2C if I2C - select SND_SOC_ADAU1701 if I2C - select SND_SOC_ADAU7002 - select SND_SOC_ADAU7118_I2C if I2C - select SND_SOC_ADAU7118_HW - select SND_SOC_ADS117X - select SND_SOC_AK4104 if SPI_MASTER - select SND_SOC_AK4118 if I2C - select SND_SOC_AK4458 if I2C - select SND_SOC_AK4535 if I2C - select SND_SOC_AK4554 - select SND_SOC_AK4613 if I2C - select SND_SOC_AK4641 if I2C - select SND_SOC_AK4642 if I2C - select SND_SOC_AK4671 if I2C - select SND_SOC_AK5386 - select SND_SOC_AK5558 if I2C - select SND_SOC_ALC5623 if I2C - select SND_SOC_ALC5632 if I2C - select SND_SOC_BT_SCO - select SND_SOC_BD28623 - select SND_SOC_CQ0093VC - select SND_SOC_CROS_EC_CODEC if CROS_EC - select SND_SOC_CS35L32 if I2C - select SND_SOC_CS35L33 if I2C - select SND_SOC_CS35L34 if I2C - select SND_SOC_CS35L35 if I2C - select SND_SOC_CS35L36 if I2C - select SND_SOC_CS42L42 if I2C - select SND_SOC_CS42L51_I2C if I2C - select SND_SOC_CS42L52 if I2C && INPUT - select SND_SOC_CS42L56 if I2C && INPUT - select SND_SOC_CS42L73 if I2C - select SND_SOC_CS4265 if I2C - select SND_SOC_CS4270 if I2C - select SND_SOC_CS4271_I2C if I2C - select SND_SOC_CS4271_SPI if SPI_MASTER - select SND_SOC_CS42XX8_I2C if I2C - select SND_SOC_CS43130 if I2C - select SND_SOC_CS4341 if SND_SOC_I2C_AND_SPI - select SND_SOC_CS4349 if I2C - select SND_SOC_CS47L15 if MFD_CS47L15 - select SND_SOC_CS47L24 if MFD_CS47L24 - select SND_SOC_CS47L35 if MFD_CS47L35 - select SND_SOC_CS47L85 if MFD_CS47L85 - select SND_SOC_CS47L90 if MFD_CS47L90 - select SND_SOC_CS47L92 if MFD_CS47L92 - select SND_SOC_CS53L30 if I2C - select SND_SOC_CX20442 if TTY - select SND_SOC_CX2072X if I2C - select SND_SOC_DA7210 if SND_SOC_I2C_AND_SPI - select SND_SOC_DA7213 if I2C - select SND_SOC_DA7218 if I2C - select SND_SOC_DA7219 if I2C - select SND_SOC_DA732X if I2C - select SND_SOC_DA9055 if I2C - select SND_SOC_DMIC if GPIOLIB - select SND_SOC_ES8316 if I2C - select SND_SOC_ES8328_SPI if SPI_MASTER - select SND_SOC_ES8328_I2C if I2C - select SND_SOC_ES7134 - select SND_SOC_ES7241 - select SND_SOC_GTM601 - select SND_SOC_HDAC_HDMI - select SND_SOC_HDAC_HDA - select SND_SOC_ICS43432 - select SND_SOC_INNO_RK3036 - select SND_SOC_ISABELLE if I2C - select SND_SOC_JZ4740_CODEC - select SND_SOC_JZ4725B_CODEC - select SND_SOC_JZ4770_CODEC - select SND_SOC_LM4857 if I2C - select SND_SOC_LM49453 if I2C - select SND_SOC_LOCHNAGAR_SC if MFD_LOCHNAGAR - select SND_SOC_MAX98088 if I2C - select SND_SOC_MAX98090 if I2C - select SND_SOC_MAX98095 if I2C - select SND_SOC_MAX98357A if GPIOLIB - select SND_SOC_MAX98371 if I2C - select SND_SOC_MAX98504 if I2C - select SND_SOC_MAX9867 if I2C - select SND_SOC_MAX98925 if I2C - select SND_SOC_MAX98926 if I2C - select SND_SOC_MAX98927 if I2C - select SND_SOC_MAX98373 if I2C - select SND_SOC_MAX9850 if I2C - select SND_SOC_MAX9860 if I2C - select SND_SOC_MAX9759 - select SND_SOC_MAX9768 if I2C - select SND_SOC_MAX9877 if I2C - select SND_SOC_MC13783 if MFD_MC13XXX - select SND_SOC_ML26124 if I2C - select SND_SOC_MT6351 if MTK_PMIC_WRAP - select SND_SOC_MT6358 if MTK_PMIC_WRAP - select SND_SOC_MT6660 if I2C - select SND_SOC_NAU8540 if I2C - select SND_SOC_NAU8810 if I2C - select SND_SOC_NAU8822 if I2C - select SND_SOC_NAU8824 if I2C - select SND_SOC_NAU8825 if I2C - select SND_SOC_HDMI_CODEC - select SND_SOC_PCM1681 if I2C - select SND_SOC_PCM1789_I2C if I2C - select SND_SOC_PCM179X_I2C if I2C - select SND_SOC_PCM179X_SPI if SPI_MASTER - select SND_SOC_PCM186X_I2C if I2C - select SND_SOC_PCM186X_SPI if SPI_MASTER - select SND_SOC_PCM3008 - select SND_SOC_PCM3060_I2C if I2C - select SND_SOC_PCM3060_SPI if SPI_MASTER - select SND_SOC_PCM3168A_I2C if I2C - select SND_SOC_PCM3168A_SPI if SPI_MASTER - select SND_SOC_PCM5102A - select SND_SOC_PCM512x_I2C if I2C - select SND_SOC_PCM512x_SPI if SPI_MASTER - select SND_SOC_RK3328 - select SND_SOC_RT274 if I2C - select SND_SOC_RT286 if I2C - select SND_SOC_RT298 if I2C - select SND_SOC_RT1011 if I2C - select SND_SOC_RT1015 if I2C - select SND_SOC_RT1305 if I2C - select SND_SOC_RT1308 if I2C - select SND_SOC_RT5514 if I2C - select SND_SOC_RT5616 if I2C - select SND_SOC_RT5631 if I2C - select SND_SOC_RT5640 if I2C - select SND_SOC_RT5645 if I2C - select SND_SOC_RT5651 if I2C - select SND_SOC_RT5659 if I2C - select SND_SOC_RT5660 if I2C - select SND_SOC_RT5663 if I2C - select SND_SOC_RT5665 if I2C - select SND_SOC_RT5668 if I2C - select SND_SOC_RT5670 if I2C - select SND_SOC_RT5677 if I2C && SPI_MASTER - select SND_SOC_RT5682 if I2C - select SND_SOC_RT700_SDW if SOUNDWIRE - select SND_SOC_RT711_SDW if SOUNDWIRE - select SND_SOC_RT715_SDW if SOUNDWIRE - select SND_SOC_RT1308_SDW if SOUNDWIRE - select SND_SOC_SGTL5000 if I2C - select SND_SOC_SI476X if MFD_SI476X_CORE - select SND_SOC_SIMPLE_AMPLIFIER - select SND_SOC_SIRF_AUDIO_CODEC - select SND_SOC_SPDIF - select SND_SOC_SSM2305 - select SND_SOC_SSM2518 if I2C - select SND_SOC_SSM2602_SPI if SPI_MASTER - select SND_SOC_SSM2602_I2C if I2C - select SND_SOC_SSM4567 if I2C - select SND_SOC_STA32X if I2C - select SND_SOC_STA350 if I2C - select SND_SOC_STA529 if I2C - select SND_SOC_STAC9766 if SND_SOC_AC97_BUS - select SND_SOC_STI_SAS - select SND_SOC_TAS2552 if I2C - select SND_SOC_TAS2562 if I2C - select SND_SOC_TAS2770 if I2C - select SND_SOC_TAS5086 if I2C - select SND_SOC_TAS571X if I2C - select SND_SOC_TAS5720 if I2C - select SND_SOC_TAS6424 if I2C - select SND_SOC_TDA7419 if I2C - select SND_SOC_TFA9879 if I2C - select SND_SOC_TLV320AIC23_I2C if I2C - select SND_SOC_TLV320AIC23_SPI if SPI_MASTER - select SND_SOC_TLV320AIC26 if SPI_MASTER - select SND_SOC_TLV320AIC31XX if I2C - select SND_SOC_TLV320AIC32X4_I2C if I2C && COMMON_CLK - select SND_SOC_TLV320AIC32X4_SPI if SPI_MASTER && COMMON_CLK - select SND_SOC_TLV320AIC3X if I2C - select SND_SOC_TPA6130A2 if I2C - select SND_SOC_TLV320DAC33 if I2C - select SND_SOC_TSCS42XX if I2C - select SND_SOC_TSCS454 if I2C - select SND_SOC_TS3A227E if I2C - select SND_SOC_TWL4030 if TWL4030_CORE - select SND_SOC_TWL6040 if TWL6040_CORE - select SND_SOC_UDA1334 if GPIOLIB - select SND_SOC_UDA134X - select SND_SOC_UDA1380 if I2C - select SND_SOC_WCD9335 if SLIMBUS - select SND_SOC_WCD934X if MFD_WCD934X && COMMON_CLK - select SND_SOC_WL1273 if MFD_WL1273_CORE - select SND_SOC_WM0010 if SPI_MASTER - select SND_SOC_WM1250_EV1 if I2C - select SND_SOC_WM2000 if I2C - select SND_SOC_WM2200 if I2C - select SND_SOC_WM5100 if I2C - select SND_SOC_WM5102 if MFD_WM5102 - select SND_SOC_WM5110 if MFD_WM5110 - select SND_SOC_WM8350 if MFD_WM8350 - select SND_SOC_WM8400 if MFD_WM8400 - select SND_SOC_WM8510 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8523 if I2C - select SND_SOC_WM8524 if GPIOLIB - select SND_SOC_WM8580 if I2C - select SND_SOC_WM8711 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8727 - select SND_SOC_WM8728 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8731 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8737 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8741 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8750 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8753 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8770 if SPI_MASTER - select SND_SOC_WM8776 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8782 - select SND_SOC_WM8804_I2C if I2C - select SND_SOC_WM8804_SPI if SPI_MASTER - select SND_SOC_WM8900 if I2C - select SND_SOC_WM8903 if I2C - select SND_SOC_WM8904 if I2C - select SND_SOC_WM8940 if I2C - select SND_SOC_WM8955 if I2C - select SND_SOC_WM8960 if I2C - select SND_SOC_WM8961 if I2C - select SND_SOC_WM8962 if I2C && INPUT - select SND_SOC_WM8971 if I2C - select SND_SOC_WM8974 if I2C - select SND_SOC_WM8978 if I2C - select SND_SOC_WM8983 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8985 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8988 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8990 if I2C - select SND_SOC_WM8991 if I2C - select SND_SOC_WM8993 if I2C - select SND_SOC_WM8994 if MFD_WM8994 - select SND_SOC_WM8995 if SND_SOC_I2C_AND_SPI - select SND_SOC_WM8996 if I2C - select SND_SOC_WM8997 if MFD_WM8997 - select SND_SOC_WM8998 if MFD_WM8998 - select SND_SOC_WM9081 if I2C - select SND_SOC_WM9090 if I2C - select SND_SOC_WM9705 if (SND_SOC_AC97_BUS || SND_SOC_AC97_BUS_NEW) - select SND_SOC_WM9712 if (SND_SOC_AC97_BUS || SND_SOC_AC97_BUS_NEW) - select SND_SOC_WM9713 if (SND_SOC_AC97_BUS || SND_SOC_AC97_BUS_NEW) - select SND_SOC_WSA881X if SOUNDWIRE + imply SND_SOC_88PM860X + imply SND_SOC_L3 + imply SND_SOC_AB8500_CODEC + imply SND_SOC_AC97_CODEC + imply SND_SOC_AD1836 + imply SND_SOC_AD193X_SPI + imply SND_SOC_AD193X_I2C + imply SND_SOC_AD1980 + imply SND_SOC_AD73311 + imply SND_SOC_ADAU1373 + imply SND_SOC_ADAU1761_I2C + imply SND_SOC_ADAU1761_SPI + imply SND_SOC_ADAU1781_I2C + imply SND_SOC_ADAU1781_SPI + imply SND_SOC_ADAV801 + imply SND_SOC_ADAV803 + imply SND_SOC_ADAU1977_SPI + imply SND_SOC_ADAU1977_I2C + imply SND_SOC_ADAU1701 + imply SND_SOC_ADAU7002 + imply SND_SOC_ADAU7118_I2C + imply SND_SOC_ADAU7118_HW + imply SND_SOC_ADS117X + imply SND_SOC_AK4104 + imply SND_SOC_AK4118 + imply SND_SOC_AK4458 + imply SND_SOC_AK4535 + imply SND_SOC_AK4554 + imply SND_SOC_AK4613 + imply SND_SOC_AK4641 + imply SND_SOC_AK4642 + imply SND_SOC_AK4671 + imply SND_SOC_AK5386 + imply SND_SOC_AK5558 + imply SND_SOC_ALC5623 + imply SND_SOC_ALC5632 + imply SND_SOC_BT_SCO + imply SND_SOC_BD28623 + imply SND_SOC_CQ0093VC + imply SND_SOC_CROS_EC_CODEC + imply SND_SOC_CS35L32 + imply SND_SOC_CS35L33 + imply SND_SOC_CS35L34 + imply SND_SOC_CS35L35 + imply SND_SOC_CS35L36 + imply SND_SOC_CS42L42 + imply SND_SOC_CS42L51_I2C + imply SND_SOC_CS42L52 + imply SND_SOC_CS42L56 + imply SND_SOC_CS42L73 + imply SND_SOC_CS4265 + imply SND_SOC_CS4270 + imply SND_SOC_CS4271_I2C + imply SND_SOC_CS4271_SPI + imply SND_SOC_CS42XX8_I2C + imply SND_SOC_CS43130 + imply SND_SOC_CS4341 + imply SND_SOC_CS4349 + imply SND_SOC_CS47L15 + imply SND_SOC_CS47L24 + imply SND_SOC_CS47L35 + imply SND_SOC_CS47L85 + imply SND_SOC_CS47L90 + imply SND_SOC_CS47L92 + imply SND_SOC_CS53L30 + imply SND_SOC_CX20442 + imply SND_SOC_CX2072X + imply SND_SOC_DA7210 + imply SND_SOC_DA7213 + imply SND_SOC_DA7218 + imply SND_SOC_DA7219 + imply SND_SOC_DA732X + imply SND_SOC_DA9055 + imply SND_SOC_DMIC + imply SND_SOC_ES8316 + imply SND_SOC_ES8328_SPI + imply SND_SOC_ES8328_I2C + imply SND_SOC_ES7134 + imply SND_SOC_ES7241 + imply SND_SOC_GTM601 + imply SND_SOC_HDAC_HDMI + imply SND_SOC_HDAC_HDA + imply SND_SOC_ICS43432 + imply SND_SOC_INNO_RK3036 + imply SND_SOC_ISABELLE + imply SND_SOC_JZ4740_CODEC + imply SND_SOC_JZ4725B_CODEC + imply SND_SOC_JZ4770_CODEC + imply SND_SOC_LM4857 + imply SND_SOC_LM49453 + imply SND_SOC_LOCHNAGAR_SC + imply SND_SOC_MAX98088 + imply SND_SOC_MAX98090 + imply SND_SOC_MAX98095 + imply SND_SOC_MAX98357A + imply SND_SOC_MAX98371 + imply SND_SOC_MAX98504 + imply SND_SOC_MAX9867 + imply SND_SOC_MAX98925 + imply SND_SOC_MAX98926 + imply SND_SOC_MAX98927 + imply SND_SOC_MAX98373 + imply SND_SOC_MAX9850 + imply SND_SOC_MAX9860 + imply SND_SOC_MAX9759 + imply SND_SOC_MAX9768 + imply SND_SOC_MAX9877 + imply SND_SOC_MC13783 + imply SND_SOC_ML26124 + imply SND_SOC_MT6351 + imply SND_SOC_MT6358 + imply SND_SOC_MT6660 + imply SND_SOC_NAU8540 + imply SND_SOC_NAU8810 + imply SND_SOC_NAU8822 + imply SND_SOC_NAU8824 + imply SND_SOC_NAU8825 + imply SND_SOC_HDMI_CODEC + imply SND_SOC_PCM1681 + imply SND_SOC_PCM1789_I2C + imply SND_SOC_PCM179X_I2C + imply SND_SOC_PCM179X_SPI + imply SND_SOC_PCM186X_I2C + imply SND_SOC_PCM186X_SPI + imply SND_SOC_PCM3008 + imply SND_SOC_PCM3060_I2C + imply SND_SOC_PCM3060_SPI + imply SND_SOC_PCM3168A_I2C + imply SND_SOC_PCM3168A_SPI + imply SND_SOC_PCM5102A + imply SND_SOC_PCM512x_I2C + imply SND_SOC_PCM512x_SPI + imply SND_SOC_RK3328 + imply SND_SOC_RT274 + imply SND_SOC_RT286 + imply SND_SOC_RT298 + imply SND_SOC_RT1011 + imply SND_SOC_RT1015 + imply SND_SOC_RT1305 + imply SND_SOC_RT1308 + imply SND_SOC_RT5514 + imply SND_SOC_RT5616 + imply SND_SOC_RT5631 + imply SND_SOC_RT5640 + imply SND_SOC_RT5645 + imply SND_SOC_RT5651 + imply SND_SOC_RT5659 + imply SND_SOC_RT5660 + imply SND_SOC_RT5663 + imply SND_SOC_RT5665 + imply SND_SOC_RT5668 + imply SND_SOC_RT5670 + imply SND_SOC_RT5677 + imply SND_SOC_RT5682 + imply SND_SOC_RT5682_SDW + imply SND_SOC_RT700_SDW + imply SND_SOC_RT711_SDW + imply SND_SOC_RT715_SDW + imply SND_SOC_RT1308_SDW + imply SND_SOC_SGTL5000 + imply SND_SOC_SI476X + imply SND_SOC_SIMPLE_AMPLIFIER + imply SND_SOC_SIRF_AUDIO_CODEC + imply SND_SOC_SPDIF + imply SND_SOC_SSM2305 + imply SND_SOC_SSM2518 + imply SND_SOC_SSM2602_SPI + imply SND_SOC_SSM2602_I2C + imply SND_SOC_SSM4567 + imply SND_SOC_STA32X + imply SND_SOC_STA350 + imply SND_SOC_STA529 + imply SND_SOC_STAC9766 + imply SND_SOC_STI_SAS + imply SND_SOC_TAS2552 + imply SND_SOC_TAS2562 + imply SND_SOC_TAS2770 + imply SND_SOC_TAS5086 + imply SND_SOC_TAS571X + imply SND_SOC_TAS5720 + imply SND_SOC_TAS6424 + imply SND_SOC_TDA7419 + imply SND_SOC_TFA9879 + imply SND_SOC_TLV320ADCX140 + imply SND_SOC_TLV320AIC23_I2C + imply SND_SOC_TLV320AIC23_SPI + imply SND_SOC_TLV320AIC26 + imply SND_SOC_TLV320AIC31XX + imply SND_SOC_TLV320AIC32X4_I2C + imply SND_SOC_TLV320AIC32X4_SPI + imply SND_SOC_TLV320AIC3X + imply SND_SOC_TPA6130A2 + imply SND_SOC_TLV320DAC33 + imply SND_SOC_TSCS42XX + imply SND_SOC_TSCS454 + imply SND_SOC_TS3A227E + imply SND_SOC_TWL4030 + imply SND_SOC_TWL6040 + imply SND_SOC_UDA1334 + imply SND_SOC_UDA134X + imply SND_SOC_UDA1380 + imply SND_SOC_WCD9335 + imply SND_SOC_WCD934X + imply SND_SOC_WL1273 + imply SND_SOC_WM0010 + imply SND_SOC_WM1250_EV1 + imply SND_SOC_WM2000 + imply SND_SOC_WM2200 + imply SND_SOC_WM5100 + imply SND_SOC_WM5102 + imply SND_SOC_WM5110 + imply SND_SOC_WM8350 + imply SND_SOC_WM8400 + imply SND_SOC_WM8510 + imply SND_SOC_WM8523 + imply SND_SOC_WM8524 + imply SND_SOC_WM8580 + imply SND_SOC_WM8711 + imply SND_SOC_WM8727 + imply SND_SOC_WM8728 + imply SND_SOC_WM8731 + imply SND_SOC_WM8737 + imply SND_SOC_WM8741 + imply SND_SOC_WM8750 + imply SND_SOC_WM8753 + imply SND_SOC_WM8770 + imply SND_SOC_WM8776 + imply SND_SOC_WM8782 + imply SND_SOC_WM8804_I2C + imply SND_SOC_WM8804_SPI + imply SND_SOC_WM8900 + imply SND_SOC_WM8903 + imply SND_SOC_WM8904 + imply SND_SOC_WM8940 + imply SND_SOC_WM8955 + imply SND_SOC_WM8960 + imply SND_SOC_WM8961 + imply SND_SOC_WM8962 + imply SND_SOC_WM8971 + imply SND_SOC_WM8974 + imply SND_SOC_WM8978 + imply SND_SOC_WM8983 + imply SND_SOC_WM8985 + imply SND_SOC_WM8988 + imply SND_SOC_WM8990 + imply SND_SOC_WM8991 + imply SND_SOC_WM8993 + imply SND_SOC_WM8994 + imply SND_SOC_WM8995 + imply SND_SOC_WM8996 + imply SND_SOC_WM8997 + imply SND_SOC_WM8998 + imply SND_SOC_WM9081 + imply SND_SOC_WM9090 + imply SND_SOC_WM9705 + imply SND_SOC_WM9712 + imply SND_SOC_WM9713 + imply SND_SOC_WSA881X help Normally ASoC codec drivers are only built if a machine driver which uses them is also built since they are only usable with a machine @@ -283,6 +285,7 @@ config SND_SOC_ALL_CODECS config SND_SOC_88PM860X tristate + depends on MFD_88PM860X config SND_SOC_ARIZONA tristate @@ -318,6 +321,7 @@ config SND_SOC_WM_ADSP config SND_SOC_AB8500_CODEC tristate + depends on ABX500_CORE config SND_SOC_AC97_CODEC tristate "Build generic ASoC AC97 CODEC driver" @@ -326,21 +330,25 @@ config SND_SOC_AC97_CODEC config SND_SOC_AD1836 tristate + depends on SPI_MASTER config SND_SOC_AD193X tristate config SND_SOC_AD193X_SPI tristate + depends on SPI_MASTER select SND_SOC_AD193X config SND_SOC_AD193X_I2C tristate + depends on I2C select SND_SOC_AD193X config SND_SOC_AD1980 - select REGMAP_AC97 tristate + depends on SND_SOC_AC97_BUS + select REGMAP_AC97 config SND_SOC_AD73311 tristate @@ -350,6 +358,7 @@ config SND_SOC_ADAU_UTILS config SND_SOC_ADAU1373 tristate + depends on I2C select SND_SOC_ADAU_UTILS config SND_SOC_ADAU1701 @@ -384,11 +393,13 @@ config SND_SOC_ADAU1781 config SND_SOC_ADAU1781_I2C tristate + depends on I2C select SND_SOC_ADAU1781 select REGMAP_I2C config SND_SOC_ADAU1781_SPI tristate + depends on SPI_MASTER select SND_SOC_ADAU1781 select REGMAP_SPI @@ -397,11 +408,13 @@ config SND_SOC_ADAU1977 config SND_SOC_ADAU1977_SPI tristate + depends on SPI_MASTER select SND_SOC_ADAU1977 select REGMAP_SPI config SND_SOC_ADAU1977_I2C tristate + depends on I2C select SND_SOC_ADAU1977 select REGMAP_I2C @@ -440,10 +453,12 @@ config SND_SOC_ADAV80X config SND_SOC_ADAV801 tristate + depends on SPI_MASTER select SND_SOC_ADAV80X config SND_SOC_ADAV803 tristate + depends on I2C select SND_SOC_ADAV80X config SND_SOC_ADS117X @@ -465,6 +480,7 @@ config SND_SOC_AK4458 config SND_SOC_AK4535 tristate + depends on I2C config SND_SOC_AK4554 tristate "AKM AK4554 CODEC" @@ -475,6 +491,7 @@ config SND_SOC_AK4613 config SND_SOC_AK4641 tristate + depends on I2C config SND_SOC_AK4642 tristate "AKM AK4642 CODEC" @@ -482,6 +499,7 @@ config SND_SOC_AK4642 config SND_SOC_AK4671 tristate + depends on I2C config SND_SOC_AK5386 tristate "AKM AK5638 CODEC" @@ -497,6 +515,7 @@ config SND_SOC_ALC5623 config SND_SOC_ALC5632 tristate + depends on I2C config SND_SOC_BD28623 tristate "ROHM BD28623 CODEC" @@ -631,6 +650,7 @@ config SND_SOC_CS47L15 config SND_SOC_CS47L24 tristate + depends on MFD_CS47L24 config SND_SOC_CS47L35 tristate @@ -697,6 +717,7 @@ config SND_SOC_L3 config SND_SOC_DA7210 tristate + depends on I2C config SND_SOC_DA7213 tristate "Dialog DA7213 CODEC" @@ -704,15 +725,19 @@ config SND_SOC_DA7213 config SND_SOC_DA7218 tristate + depends on I2C config SND_SOC_DA7219 tristate + depends on I2C config SND_SOC_DA732X tristate + depends on I2C config SND_SOC_DA9055 tristate + depends on I2C config SND_SOC_DMIC tristate "Generic Digital Microphone CODEC" @@ -772,9 +797,11 @@ config SND_SOC_INNO_RK3036 config SND_SOC_ISABELLE tristate + depends on I2C config SND_SOC_LM49453 tristate + depends on I2C config SND_SOC_LOCHNAGAR_SC tristate "Lochnagar Sound Card" @@ -801,17 +828,20 @@ config SND_SOC_MAX98088 depends on I2C config SND_SOC_MAX98090 - tristate + tristate + depends on I2C config SND_SOC_MAX98095 - tristate + tristate + depends on I2C config SND_SOC_MAX98357A tristate "Maxim MAX98357A CODEC" depends on GPIOLIB config SND_SOC_MAX98371 - tristate + tristate + depends on I2C config SND_SOC_MAX98504 tristate "Maxim MAX98504 speaker amplifier" @@ -822,10 +852,12 @@ config SND_SOC_MAX9867 depends on I2C config SND_SOC_MAX98925 - tristate + tristate + depends on I2C config SND_SOC_MAX98926 tristate + depends on I2C config SND_SOC_MAX98927 tristate "Maxim Integrated MAX98927 Speaker Amplifier" @@ -837,6 +869,7 @@ config SND_SOC_MAX98373 config SND_SOC_MAX9850 tristate + depends on I2C config SND_SOC_MAX9860 tristate "Maxim MAX9860 Mono Audio Voice Codec" @@ -1015,26 +1048,32 @@ config SND_SOC_RT298 config SND_SOC_RT1011 tristate + depends on I2C config SND_SOC_RT1015 tristate + depends on I2C config SND_SOC_RT1305 tristate + depends on I2C config SND_SOC_RT1308 tristate + depends on I2C config SND_SOC_RT1308_SDW tristate "Realtek RT1308 Codec - SDW" - depends on SOUNDWIRE + depends on I2C && SOUNDWIRE select REGMAP_SOUNDWIRE config SND_SOC_RT5514 tristate + depends on I2C config SND_SOC_RT5514_SPI tristate + depends on SPI_MASTER config SND_SOC_RT5514_SPI_BUILTIN bool # force RT5514_SPI to be built-in to avoid link errors @@ -1050,33 +1089,43 @@ config SND_SOC_RT5631 config SND_SOC_RT5640 tristate + depends on I2C config SND_SOC_RT5645 tristate + depends on I2C config SND_SOC_RT5651 tristate + depends on I2C config SND_SOC_RT5659 tristate + depends on I2C config SND_SOC_RT5660 tristate + depends on I2C config SND_SOC_RT5663 tristate + depends on I2C config SND_SOC_RT5665 tristate + depends on I2C config SND_SOC_RT5668 tristate + depends on I2C config SND_SOC_RT5670 tristate + depends on I2C config SND_SOC_RT5677 tristate + depends on I2C select REGMAP_I2C select REGMAP_IRQ @@ -1086,6 +1135,13 @@ config SND_SOC_RT5677_SPI config SND_SOC_RT5682 tristate + depends on I2C || SOUNDWIRE + +config SND_SOC_RT5682_SDW + tristate "Realtek RT5682 Codec - SDW" + depends on SOUNDWIRE + select SND_SOC_RT5682 + select REGMAP_SOUNDWIRE config SND_SOC_RT700 tristate @@ -1153,6 +1209,7 @@ config SND_SOC_SSM2305 config SND_SOC_SSM2518 tristate + depends on I2C config SND_SOC_SSM2602 tristate @@ -1184,9 +1241,11 @@ config SND_SOC_STA350 config SND_SOC_STA529 tristate + depends on I2C config SND_SOC_STAC9766 tristate + depends on SND_SOC_AC97_BUS config SND_SOC_STI_SAS tristate "codec Audio support for STI SAS codec" @@ -1281,6 +1340,15 @@ config SND_SOC_TLV320AIC3X config SND_SOC_TLV320DAC33 tristate + depends on I2C + +config SND_SOC_TLV320ADCX140 + tristate "Texas Instruments TLV320ADCX140 CODEC family" + depends on I2C + select REGMAP_I2C + help + Add support for Texas Instruments tlv320adc3140, tlv320adc5140 and + tlv320adc6140 quad channel ADCs. config SND_SOC_TS3A227E tristate "TI Headset/Mic detect and keypress chip" @@ -1301,11 +1369,13 @@ config SND_SOC_TSCS454 Add support for Tempo Semiconductor's TSCS454 audio CODEC. config SND_SOC_TWL4030 - select MFD_TWL4030_AUDIO tristate + depends on TWL4030_CORE + select MFD_TWL4030_AUDIO config SND_SOC_TWL6040 tristate + depends on TWL6040_CORE config SND_SOC_UDA1334 tristate "NXP UDA1334 DAC" @@ -1345,30 +1415,40 @@ config SND_SOC_WL1273 config SND_SOC_WM0010 tristate + depends on SPI_MASTER config SND_SOC_WM1250_EV1 tristate + depends on I2C config SND_SOC_WM2000 tristate + depends on I2C config SND_SOC_WM2200 tristate + depends on I2C config SND_SOC_WM5100 tristate + depends on I2C config SND_SOC_WM5102 tristate + depends on MFD_WM5102 config SND_SOC_WM5110 tristate + depends on MFD_WM5110 config SND_SOC_WM8350 tristate + depends on MFD_WM8350 config SND_SOC_WM8400 tristate + # FIXME nothing selects SND_SOC_WM8400?? + depends on MFD_WM8400 config SND_SOC_WM8510 tristate "Wolfson Microelectronics WM8510 CODEC" @@ -1456,9 +1536,11 @@ config SND_SOC_WM8904 config SND_SOC_WM8940 tristate + depends on I2C config SND_SOC_WM8955 tristate + depends on I2C config SND_SOC_WM8960 tristate "Wolfson Microelectronics WM8960 CODEC" @@ -1466,6 +1548,7 @@ config SND_SOC_WM8960 config SND_SOC_WM8961 tristate + depends on I2C config SND_SOC_WM8962 tristate "Wolfson Microelectronics WM8962 CODEC" @@ -1473,6 +1556,7 @@ config SND_SOC_WM8962 config SND_SOC_WM8971 tristate + depends on I2C config SND_SOC_WM8974 tristate "Wolfson Microelectronics WM8974 codec" @@ -1484,6 +1568,7 @@ config SND_SOC_WM8978 config SND_SOC_WM8983 tristate + depends on I2C config SND_SOC_WM8985 tristate "Wolfson Microelectronics WM8985 and WM8758 codec driver" @@ -1494,12 +1579,15 @@ config SND_SOC_WM8988 config SND_SOC_WM8990 tristate + depends on I2C config SND_SOC_WM8991 tristate + depends on I2C config SND_SOC_WM8993 tristate + depends on I2C config SND_SOC_WM8994 tristate @@ -1509,12 +1597,15 @@ config SND_SOC_WM8995 config SND_SOC_WM8996 tristate + depends on I2C config SND_SOC_WM8997 tristate + depends on MFD_WM8997 config SND_SOC_WM8998 tristate + depends on MFD_WM8998 config SND_SOC_WM9081 tristate @@ -1522,19 +1613,23 @@ config SND_SOC_WM9081 config SND_SOC_WM9090 tristate + depends on I2C config SND_SOC_WM9705 tristate + depends on SND_SOC_AC97_BUS select REGMAP_AC97 select AC97_BUS_COMPAT if AC97_BUS_NEW config SND_SOC_WM9712 tristate + depends on SND_SOC_AC97_BUS select REGMAP_AC97 select AC97_BUS_COMPAT if AC97_BUS_NEW config SND_SOC_WM9713 tristate + depends on SND_SOC_AC97_BUS select REGMAP_AC97 select AC97_BUS_COMPAT if AC97_BUS_NEW @@ -1555,6 +1650,7 @@ config SND_SOC_ZX_AUD96P22 # Amp config SND_SOC_LM4857 tristate + depends on I2C config SND_SOC_MAX9759 tristate "Maxim MAX9759 speaker Amplifier" @@ -1562,15 +1658,19 @@ config SND_SOC_MAX9759 config SND_SOC_MAX9768 tristate + depends on I2C config SND_SOC_MAX9877 tristate + depends on I2C config SND_SOC_MC13783 tristate + depends on MFD_MC13XXX config SND_SOC_ML26124 tristate + depends on I2C config SND_SOC_MT6351 tristate "MediaTek MT6351 Codec" @@ -1608,6 +1708,7 @@ config SND_SOC_NAU8824 config SND_SOC_NAU8825 tristate + depends on I2C config SND_SOC_TPA6130A2 tristate "Texas Instruments TPA6130A2 headphone amplifier" diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile index ba1b4b3fa2da..03533157cda6 100644 --- a/sound/soc/codecs/Makefile +++ b/sound/soc/codecs/Makefile @@ -177,6 +177,7 @@ snd-soc-rt5670-objs := rt5670.o snd-soc-rt5677-objs := rt5677.o snd-soc-rt5677-spi-objs := rt5677-spi.o snd-soc-rt5682-objs := rt5682.o +snd-soc-rt5682-sdw-objs := rt5682-sdw.o snd-soc-rt700-objs := rt700.o rt700-sdw.o snd-soc-rt711-objs := rt711.o rt711-sdw.o snd-soc-rt715-objs := rt715.o rt715-sdw.o @@ -218,6 +219,7 @@ snd-soc-tlv320aic32x4-i2c-objs := tlv320aic32x4-i2c.o snd-soc-tlv320aic32x4-spi-objs := tlv320aic32x4-spi.o snd-soc-tlv320aic3x-objs := tlv320aic3x.o snd-soc-tlv320dac33-objs := tlv320dac33.o +snd-soc-tlv320adcx140-objs := tlv320adcx140.o snd-soc-tscs42xx-objs := tscs42xx.o snd-soc-tscs454-objs := tscs454.o snd-soc-ts3a227e-objs := ts3a227e.o @@ -476,6 +478,7 @@ obj-$(CONFIG_SND_SOC_RT5670) += snd-soc-rt5670.o obj-$(CONFIG_SND_SOC_RT5677) += snd-soc-rt5677.o obj-$(CONFIG_SND_SOC_RT5677_SPI) += snd-soc-rt5677-spi.o obj-$(CONFIG_SND_SOC_RT5682) += snd-soc-rt5682.o +obj-$(CONFIG_SND_SOC_RT5682_SDW) += snd-soc-rt5682-sdw.o obj-$(CONFIG_SND_SOC_RT700) += snd-soc-rt700.o obj-$(CONFIG_SND_SOC_RT711) += snd-soc-rt711.o obj-$(CONFIG_SND_SOC_RT715) += snd-soc-rt715.o @@ -516,6 +519,7 @@ obj-$(CONFIG_SND_SOC_TLV320AIC32X4_I2C) += snd-soc-tlv320aic32x4-i2c.o obj-$(CONFIG_SND_SOC_TLV320AIC32X4_SPI) += snd-soc-tlv320aic32x4-spi.o obj-$(CONFIG_SND_SOC_TLV320AIC3X) += snd-soc-tlv320aic3x.o obj-$(CONFIG_SND_SOC_TLV320DAC33) += snd-soc-tlv320dac33.o +obj-$(CONFIG_SND_SOC_TLV320ADCX140) += snd-soc-tlv320adcx140.o obj-$(CONFIG_SND_SOC_TSCS42XX) += snd-soc-tscs42xx.o obj-$(CONFIG_SND_SOC_TSCS454) += snd-soc-tscs454.o obj-$(CONFIG_SND_SOC_TS3A227E) += snd-soc-ts3a227e.o diff --git a/sound/soc/codecs/cros_ec_codec.c b/sound/soc/codecs/cros_ec_codec.c index 6a24f570c5e8..d3dc42aa6825 100644 --- a/sound/soc/codecs/cros_ec_codec.c +++ b/sound/soc/codecs/cros_ec_codec.c @@ -45,6 +45,9 @@ struct cros_ec_codec_priv { /* DMIC */ atomic_t dmic_probed; + /* I2S_RX */ + uint32_t i2s_rx_bclk_ratio; + /* WoV */ bool wov_enabled; uint8_t *wov_audio_shm_p; @@ -259,6 +262,7 @@ static int i2s_rx_hw_params(struct snd_pcm_substream *substream, snd_soc_component_get_drvdata(component); struct ec_param_ec_codec_i2s_rx p; enum ec_codec_i2s_rx_sample_depth depth; + uint32_t bclk; int ret; if (params_rate(params) != 48000) @@ -284,15 +288,29 @@ static int i2s_rx_hw_params(struct snd_pcm_substream *substream, if (ret < 0) return ret; - dev_dbg(component->dev, "set bclk to %u\n", - snd_soc_params_to_bclk(params)); + if (priv->i2s_rx_bclk_ratio) + bclk = params_rate(params) * priv->i2s_rx_bclk_ratio; + else + bclk = snd_soc_params_to_bclk(params); + + dev_dbg(component->dev, "set bclk to %u\n", bclk); p.cmd = EC_CODEC_I2S_RX_SET_BCLK; - p.set_bclk_param.bclk = snd_soc_params_to_bclk(params); + p.set_bclk_param.bclk = bclk; return send_ec_host_command(priv->ec_device, EC_CMD_EC_CODEC_I2S_RX, (uint8_t *)&p, sizeof(p), NULL, 0); } +static int i2s_rx_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) +{ + struct snd_soc_component *component = dai->component; + struct cros_ec_codec_priv *priv = + snd_soc_component_get_drvdata(component); + + priv->i2s_rx_bclk_ratio = ratio; + return 0; +} + static int i2s_rx_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) { struct snd_soc_component *component = dai->component; @@ -340,6 +358,7 @@ static int i2s_rx_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) static const struct snd_soc_dai_ops i2s_rx_dai_ops = { .hw_params = i2s_rx_hw_params, .set_fmt = i2s_rx_set_fmt, + .set_bclk_ratio = i2s_rx_set_bclk_ratio, }; static int i2s_rx_event(struct snd_soc_dapm_widget *w, diff --git a/sound/soc/codecs/cs4271.c b/sound/soc/codecs/cs4271.c index 04b86a51e055..62f412d6f9f2 100644 --- a/sound/soc/codecs/cs4271.c +++ b/sound/soc/codecs/cs4271.c @@ -356,9 +356,9 @@ static int cs4271_hw_params(struct snd_pcm_substream *substream, */ if ((substream->stream == SNDRV_PCM_STREAM_PLAYBACK && - !dai->capture_active) || + !dai->stream_active[SNDRV_PCM_STREAM_CAPTURE]) || (substream->stream == SNDRV_PCM_STREAM_CAPTURE && - !dai->playback_active)) { + !dai->stream_active[SNDRV_PCM_STREAM_PLAYBACK])) { ret = regmap_update_bits(cs4271->regmap, CS4271_MODE2, CS4271_MODE2_PDN, CS4271_MODE2_PDN); diff --git a/sound/soc/codecs/cs47l15.c b/sound/soc/codecs/cs47l15.c index e8840dc142ef..8d1869bf7f9c 100644 --- a/sound/soc/codecs/cs47l15.c +++ b/sound/soc/codecs/cs47l15.c @@ -1239,12 +1239,12 @@ static int cs47l15_open(struct snd_compr_stream *stream) struct madera *madera = priv->madera; int n_adsp; - if (strcmp(rtd->codec_dai->name, "cs47l15-dsp-trace") == 0) { + if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l15-dsp-trace") == 0) { n_adsp = 0; } else { dev_err(madera->dev, "No suitable compressed stream for DAI '%s'\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); return -EINVAL; } diff --git a/sound/soc/codecs/cs47l24.c b/sound/soc/codecs/cs47l24.c index 25bffc2968f0..6b0570f59630 100644 --- a/sound/soc/codecs/cs47l24.c +++ b/sound/soc/codecs/cs47l24.c @@ -1076,14 +1076,14 @@ static int cs47l24_open(struct snd_compr_stream *stream) struct arizona *arizona = priv->core.arizona; int n_adsp; - if (strcmp(rtd->codec_dai->name, "cs47l24-dsp-voicectrl") == 0) { + if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l24-dsp-voicectrl") == 0) { n_adsp = 2; - } else if (strcmp(rtd->codec_dai->name, "cs47l24-dsp-trace") == 0) { + } else if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l24-dsp-trace") == 0) { n_adsp = 1; } else { dev_err(arizona->dev, "No suitable compressed stream for DAI '%s'\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); return -EINVAL; } diff --git a/sound/soc/codecs/cs47l35.c b/sound/soc/codecs/cs47l35.c index 3d48a0d9ecc5..18839807c9d1 100644 --- a/sound/soc/codecs/cs47l35.c +++ b/sound/soc/codecs/cs47l35.c @@ -1514,14 +1514,14 @@ static int cs47l35_open(struct snd_compr_stream *stream) struct madera *madera = priv->madera; int n_adsp; - if (strcmp(rtd->codec_dai->name, "cs47l35-dsp-voicectrl") == 0) { + if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l35-dsp-voicectrl") == 0) { n_adsp = 2; - } else if (strcmp(rtd->codec_dai->name, "cs47l35-dsp-trace") == 0) { + } else if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l35-dsp-trace") == 0) { n_adsp = 0; } else { dev_err(madera->dev, "No suitable compressed stream for DAI '%s'\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); return -EINVAL; } diff --git a/sound/soc/codecs/cs47l85.c b/sound/soc/codecs/cs47l85.c index bef3471f482d..a575113207f0 100644 --- a/sound/soc/codecs/cs47l85.c +++ b/sound/soc/codecs/cs47l85.c @@ -2457,14 +2457,14 @@ static int cs47l85_open(struct snd_compr_stream *stream) struct madera *madera = priv->madera; int n_adsp; - if (strcmp(rtd->codec_dai->name, "cs47l85-dsp-voicectrl") == 0) { + if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l85-dsp-voicectrl") == 0) { n_adsp = 5; - } else if (strcmp(rtd->codec_dai->name, "cs47l85-dsp-trace") == 0) { + } else if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l85-dsp-trace") == 0) { n_adsp = 0; } else { dev_err(madera->dev, "No suitable compressed stream for DAI '%s'\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); return -EINVAL; } diff --git a/sound/soc/codecs/cs47l90.c b/sound/soc/codecs/cs47l90.c index 266eade82764..81a1311b14e6 100644 --- a/sound/soc/codecs/cs47l90.c +++ b/sound/soc/codecs/cs47l90.c @@ -2368,14 +2368,14 @@ static int cs47l90_open(struct snd_compr_stream *stream) struct madera *madera = priv->madera; int n_adsp; - if (strcmp(rtd->codec_dai->name, "cs47l90-dsp-voicectrl") == 0) { + if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l90-dsp-voicectrl") == 0) { n_adsp = 5; - } else if (strcmp(rtd->codec_dai->name, "cs47l90-dsp-trace") == 0) { + } else if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l90-dsp-trace") == 0) { n_adsp = 0; } else { dev_err(madera->dev, "No suitable compressed stream for DAI '%s'\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); return -EINVAL; } diff --git a/sound/soc/codecs/cs47l92.c b/sound/soc/codecs/cs47l92.c index 942040fd354f..15fc213d178d 100644 --- a/sound/soc/codecs/cs47l92.c +++ b/sound/soc/codecs/cs47l92.c @@ -1840,12 +1840,12 @@ static int cs47l92_open(struct snd_compr_stream *stream) struct madera *madera = priv->madera; int n_adsp; - if (strcmp(rtd->codec_dai->name, "cs47l92-dsp-trace") == 0) { + if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "cs47l92-dsp-trace") == 0) { n_adsp = 0; } else { dev_err(madera->dev, "No suitable compressed stream for DAI '%s'\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); return -EINVAL; } diff --git a/sound/soc/codecs/hdac_hdmi.c b/sound/soc/codecs/hdac_hdmi.c index e6558475e006..fba9b749839d 100644 --- a/sound/soc/codecs/hdac_hdmi.c +++ b/sound/soc/codecs/hdac_hdmi.c @@ -1998,11 +1998,11 @@ static struct hdac_hdmi_drv_data intel_drv_data = { static int hdac_hdmi_dev_probe(struct hdac_device *hdev) { - struct hdac_hdmi_priv *hdmi_priv = NULL; + struct hdac_hdmi_priv *hdmi_priv; struct snd_soc_dai_driver *hdmi_dais = NULL; - struct hdac_ext_link *hlink = NULL; + struct hdac_ext_link *hlink; int num_dais = 0; - int ret = 0; + int ret; struct hdac_driver *hdrv = drv_to_hdac_driver(hdev->dev.driver); const struct hda_device_id *hdac_id = hdac_get_device_id(hdev, hdrv); diff --git a/sound/soc/codecs/max98357a.c b/sound/soc/codecs/max98357a.c index 16313b973eaa..a8bd793a7867 100644 --- a/sound/soc/codecs/max98357a.c +++ b/sound/soc/codecs/max98357a.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -24,26 +25,24 @@ struct max98357a_priv { unsigned int sdmode_delay; }; -static int max98357a_daiops_trigger(struct snd_pcm_substream *substream, - int cmd, struct snd_soc_dai *dai) +static int max98357a_sdmode_event(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *kcontrol, int event) { - struct max98357a_priv *max98357a = snd_soc_dai_get_drvdata(dai); + struct snd_soc_component *component = + snd_soc_dapm_to_component(w->dapm); + struct max98357a_priv *max98357a = + snd_soc_component_get_drvdata(component); if (!max98357a->sdmode) return 0; - switch (cmd) { - case SNDRV_PCM_TRIGGER_START: - case SNDRV_PCM_TRIGGER_RESUME: - case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: - mdelay(max98357a->sdmode_delay); + if (event & SND_SOC_DAPM_POST_PMU) { + msleep(max98357a->sdmode_delay); gpiod_set_value(max98357a->sdmode, 1); - break; - case SNDRV_PCM_TRIGGER_STOP: - case SNDRV_PCM_TRIGGER_SUSPEND: - case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + dev_dbg(component->dev, "set sdmode to 1"); + } else if (event & SND_SOC_DAPM_PRE_PMD) { gpiod_set_value(max98357a->sdmode, 0); - break; + dev_dbg(component->dev, "set sdmode to 0"); } return 0; @@ -51,10 +50,14 @@ static int max98357a_daiops_trigger(struct snd_pcm_substream *substream, static const struct snd_soc_dapm_widget max98357a_dapm_widgets[] = { SND_SOC_DAPM_OUTPUT("Speaker"), + SND_SOC_DAPM_OUT_DRV_E("SD_MODE", SND_SOC_NOPM, 0, 0, NULL, 0, + max98357a_sdmode_event, + SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD), }; static const struct snd_soc_dapm_route max98357a_dapm_routes[] = { - {"Speaker", NULL, "HiFi Playback"}, + {"SD_MODE", NULL, "HiFi Playback"}, + {"Speaker", NULL, "SD_MODE"}, }; static const struct snd_soc_component_driver max98357a_component_driver = { @@ -68,10 +71,6 @@ static const struct snd_soc_component_driver max98357a_component_driver = { .non_legacy_dai_naming = 1, }; -static const struct snd_soc_dai_ops max98357a_dai_ops = { - .trigger = max98357a_daiops_trigger, -}; - static struct snd_soc_dai_driver max98357a_dai_driver = { .name = "HiFi", .playback = { @@ -91,7 +90,6 @@ static struct snd_soc_dai_driver max98357a_dai_driver = { .channels_min = 1, .channels_max = 2, }, - .ops = &max98357a_dai_ops, }; static int max98357a_platform_probe(struct platform_device *pdev) @@ -135,6 +133,7 @@ MODULE_DEVICE_TABLE(of, max98357a_device_id); #ifdef CONFIG_ACPI static const struct acpi_device_id max98357a_acpi_match[] = { { "MX98357A", 0 }, + { "MX98360A", 0 }, {}, }; MODULE_DEVICE_TABLE(acpi, max98357a_acpi_match); diff --git a/sound/soc/codecs/mt6660.c b/sound/soc/codecs/mt6660.c index a36c416caad4..d1797003c83d 100644 --- a/sound/soc/codecs/mt6660.c +++ b/sound/soc/codecs/mt6660.c @@ -1,15 +1,13 @@ -// SPDX-License-Identifier: GPL-2.0 // +// SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2019 MediaTek Inc. #include #include -#include #include #include #include #include -#include #include #include #include @@ -225,14 +223,87 @@ static int _mt6660_chip_power_on(struct mt6660_chip *chip, int on_off) 0x01, on_off ? 0x00 : 0x01); } +struct reg_table { + uint32_t addr; + uint32_t mask; + uint32_t val; +}; + +static const struct reg_table mt6660_setting_table[] = { + { 0x20, 0x80, 0x00 }, + { 0x30, 0x01, 0x00 }, + { 0x50, 0x1c, 0x04 }, + { 0xB1, 0x0c, 0x00 }, + { 0xD3, 0x03, 0x03 }, + { 0xE0, 0x01, 0x00 }, + { 0x98, 0x44, 0x04 }, + { 0xB9, 0xff, 0x82 }, + { 0xB7, 0x7777, 0x7273 }, + { 0xB6, 0x07, 0x03 }, + { 0x6B, 0xe0, 0x20 }, + { 0x07, 0xff, 0x70 }, + { 0xBB, 0xff, 0x20 }, + { 0x69, 0xff, 0x40 }, + { 0xBD, 0xffff, 0x17f8 }, + { 0x70, 0xff, 0x15 }, + { 0x7C, 0xff, 0x00 }, + { 0x46, 0xff, 0x1d }, + { 0x1A, 0xffffffff, 0x7fdb7ffe }, + { 0x1B, 0xffffffff, 0x7fdb7ffe }, + { 0x51, 0xff, 0x58 }, + { 0xA2, 0xff, 0xce }, + { 0x33, 0xffff, 0x7fff }, + { 0x4C, 0xffff, 0x0116 }, + { 0x16, 0x1800, 0x0800 }, + { 0x68, 0x1f, 0x07 }, +}; + +static int mt6660_component_setting(struct snd_soc_component *component) +{ + struct mt6660_chip *chip = snd_soc_component_get_drvdata(component); + int ret = 0; + size_t i = 0; + + ret = _mt6660_chip_power_on(chip, 1); + if (ret < 0) { + dev_err(component->dev, "%s chip power on failed\n", __func__); + return ret; + } + + for (i = 0; i < ARRAY_SIZE(mt6660_setting_table); i++) { + ret = snd_soc_component_update_bits(component, + mt6660_setting_table[i].addr, + mt6660_setting_table[i].mask, + mt6660_setting_table[i].val); + if (ret < 0) { + dev_err(component->dev, "%s update 0x%02x failed\n", + __func__, mt6660_setting_table[i].addr); + return ret; + } + } + + ret = _mt6660_chip_power_on(chip, 0); + if (ret < 0) { + dev_err(component->dev, "%s chip power off failed\n", __func__); + return ret; + } + + return 0; +} + static int mt6660_component_probe(struct snd_soc_component *component) { struct mt6660_chip *chip = snd_soc_component_get_drvdata(component); + int ret; dev_dbg(component->dev, "%s\n", __func__); snd_soc_component_init_regmap(component, chip->regmap); - return 0; + ret = mt6660_component_setting(component); + if (ret < 0) + dev_err(chip->dev, "mt6660 component setting failed\n"); + + return ret; } static void mt6660_component_remove(struct snd_soc_component *component) @@ -506,4 +577,4 @@ module_i2c_driver(mt6660_i2c_driver); MODULE_AUTHOR("Jeff Chang "); MODULE_DESCRIPTION("MT6660 SPKAMP Driver"); MODULE_LICENSE("GPL"); -MODULE_VERSION("1.0.7_G"); +MODULE_VERSION("1.0.8_G"); diff --git a/sound/soc/codecs/rk3328_codec.c b/sound/soc/codecs/rk3328_codec.c index 287c962ba00d..115706a55577 100644 --- a/sound/soc/codecs/rk3328_codec.c +++ b/sound/soc/codecs/rk3328_codec.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,7 @@ struct rk3328_codec_priv { struct regmap *regmap; - struct regmap *grf; + struct gpio_desc *mute; struct clk *mclk; struct clk *pclk; unsigned int sclk; @@ -106,16 +107,6 @@ static int rk3328_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) return 0; } -static void rk3328_analog_output(struct rk3328_codec_priv *rk3328, int mute) -{ - unsigned int val = BIT(17); - - if (mute) - val |= BIT(1); - - regmap_write(rk3328->grf, RK3328_GRF_SOC_CON10, val); -} - static int rk3328_digital_mute(struct snd_soc_dai *dai, int mute) { struct rk3328_codec_priv *rk3328 = @@ -205,7 +196,7 @@ static int rk3328_codec_open_playback(struct rk3328_codec_priv *rk3328) } msleep(rk3328->spk_depop_time); - rk3328_analog_output(rk3328, 1); + gpiod_set_value(rk3328->mute, 0); regmap_update_bits(rk3328->regmap, HPOUTL_GAIN_CTRL, HPOUTL_GAIN_MASK, OUT_VOLUME); @@ -246,7 +237,7 @@ static int rk3328_codec_close_playback(struct rk3328_codec_priv *rk3328) { size_t i; - rk3328_analog_output(rk3328, 0); + gpiod_set_value(rk3328->mute, 1); regmap_update_bits(rk3328->regmap, HPOUTL_GAIN_CTRL, HPOUTL_GAIN_MASK, 0); @@ -446,7 +437,6 @@ static int rk3328_platform_probe(struct platform_device *pdev) dev_err(&pdev->dev, "missing 'rockchip,grf'\n"); return PTR_ERR(grf); } - rk3328->grf = grf; /* enable i2s_acodec_en */ regmap_write(grf, RK3328_GRF_SOC_CON2, (BIT(14) << 16 | BIT(14))); @@ -458,7 +448,18 @@ static int rk3328_platform_probe(struct platform_device *pdev) rk3328->spk_depop_time = 200; } - rk3328_analog_output(rk3328, 0); + rk3328->mute = gpiod_get_optional(&pdev->dev, "mute", GPIOD_OUT_HIGH); + if (IS_ERR(rk3328->mute)) + return PTR_ERR(rk3328->mute); + /* + * Rock64 is the only supported platform to have widely relied on + * this; if we do happen to come across an old DTB, just leave the + * external mute forced off. + */ + if (!rk3328->mute && of_machine_is_compatible("pine64,rock64")) { + dev_warn(&pdev->dev, "assuming implicit control of GPIO_MUTE; update devicetree if possible\n"); + regmap_write(grf, RK3328_GRF_SOC_CON10, BIT(17) | BIT(1)); + } rk3328->mclk = devm_clk_get(&pdev->dev, "mclk"); if (IS_ERR(rk3328->mclk)) diff --git a/sound/soc/codecs/rl6231.c b/sound/soc/codecs/rl6231.c index a887d5ccb10d..d181c217d835 100644 --- a/sound/soc/codecs/rl6231.c +++ b/sound/soc/codecs/rl6231.c @@ -102,6 +102,7 @@ struct pll_calc_map { static const struct pll_calc_map pll_preset_table[] = { {19200000, 4096000, 23, 14, 1, false}, {19200000, 24576000, 3, 30, 3, false}, + {3840000, 24576000, 3, 30, 0, true}, }; static unsigned int find_best_div(unsigned int in, diff --git a/sound/soc/codecs/rl6231.h b/sound/soc/codecs/rl6231.h index 31a9643b0afd..6d8ed0377296 100644 --- a/sound/soc/codecs/rl6231.h +++ b/sound/soc/codecs/rl6231.h @@ -10,7 +10,7 @@ #ifndef __RL6231_H__ #define __RL6231_H__ -#define RL6231_PLL_INP_MAX 40000000 +#define RL6231_PLL_INP_MAX 50000000 #define RL6231_PLL_INP_MIN 256000 #define RL6231_PLL_N_MAX 0x1ff #define RL6231_PLL_K_MAX 0x1f diff --git a/sound/soc/codecs/rt1015.c b/sound/soc/codecs/rt1015.c index 66eb55b4ffd4..bb310bc7febd 100644 --- a/sound/soc/codecs/rt1015.c +++ b/sound/soc/codecs/rt1015.c @@ -444,7 +444,7 @@ static int rt1015_boost_mode_put(struct snd_kcontrol *kcontrol, return 0; } -static int rt5518_bypass_boost_get(struct snd_kcontrol *kcontrol, +static int rt1015_bypass_boost_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { struct snd_soc_component *component = @@ -457,7 +457,7 @@ static int rt5518_bypass_boost_get(struct snd_kcontrol *kcontrol, return 0; } -static int rt5518_bypass_boost_put(struct snd_kcontrol *kcontrol, +static int rt1015_bypass_boost_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { struct snd_soc_component *component = @@ -497,7 +497,7 @@ static const struct snd_kcontrol_new rt1015_snd_controls[] = { rt1015_boost_mode_get, rt1015_boost_mode_put), SOC_ENUM("Mono LR Select", rt1015_mono_lr_sel), SOC_SINGLE_EXT("Bypass Boost", SND_SOC_NOPM, 0, 1, 0, - rt5518_bypass_boost_get, rt5518_bypass_boost_put), + rt1015_bypass_boost_get, rt1015_bypass_boost_put), }; static int rt1015_is_sys_clk_from_pll(struct snd_soc_dapm_widget *source, @@ -841,12 +841,12 @@ static void rt1015_remove(struct snd_soc_component *component) #define RT1015_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE | \ SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S8) -struct snd_soc_dai_ops rt1015_aif_dai_ops = { +static struct snd_soc_dai_ops rt1015_aif_dai_ops = { .hw_params = rt1015_hw_params, .set_fmt = rt1015_set_dai_fmt, }; -struct snd_soc_dai_driver rt1015_dai[] = { +static struct snd_soc_dai_driver rt1015_dai[] = { { .name = "rt1015-aif", .id = 0, diff --git a/sound/soc/codecs/rt1308-sdw.c b/sound/soc/codecs/rt1308-sdw.c index d930f60cb797..a5a7e46de246 100644 --- a/sound/soc/codecs/rt1308-sdw.c +++ b/sound/soc/codecs/rt1308-sdw.c @@ -507,6 +507,28 @@ static void rt1308_sdw_shutdown(struct snd_pcm_substream *substream, kfree(stream); } +static int rt1308_sdw_set_tdm_slot(struct snd_soc_dai *dai, + unsigned int tx_mask, + unsigned int rx_mask, + int slots, int slot_width) +{ + struct snd_soc_component *component = dai->component; + struct rt1308_sdw_priv *rt1308 = + snd_soc_component_get_drvdata(component); + + if (tx_mask) + return -EINVAL; + + if (slots > 2) + return -EINVAL; + + rt1308->rx_mask = rx_mask; + rt1308->slots = slots; + /* slot_width is not used since it's irrelevant for SoundWire */ + + return 0; +} + static int rt1308_sdw_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) { @@ -517,7 +539,7 @@ static int rt1308_sdw_hw_params(struct snd_pcm_substream *substream, struct sdw_port_config port_config; enum sdw_data_direction direction; struct sdw_stream_data *stream; - int retval, port, num_channels; + int retval, port, num_channels, ch_mask; dev_dbg(dai->dev, "%s %s", __func__, dai->name); stream = snd_soc_dai_get_dma_data(dai, substream); @@ -537,13 +559,20 @@ static int rt1308_sdw_hw_params(struct snd_pcm_substream *substream, return -EINVAL; } + if (rt1308->slots) { + num_channels = rt1308->slots; + ch_mask = rt1308->rx_mask; + } else { + num_channels = params_channels(params); + ch_mask = (1 << num_channels) - 1; + } + stream_config.frame_rate = params_rate(params); - stream_config.ch_count = params_channels(params); + stream_config.ch_count = num_channels; stream_config.bps = snd_pcm_format_width(params_format(params)); stream_config.direction = direction; - num_channels = params_channels(params); - port_config.ch_mask = (1 << (num_channels)) - 1; + port_config.ch_mask = ch_mask; port_config.num = port; retval = sdw_stream_add_slave(rt1308->sdw_slave, &stream_config, @@ -597,6 +626,7 @@ static const struct snd_soc_dai_ops rt1308_aif_dai_ops = { .hw_free = rt1308_sdw_pcm_hw_free, .set_sdw_stream = rt1308_set_sdw_stream, .shutdown = rt1308_sdw_shutdown, + .set_tdm_slot = rt1308_sdw_set_tdm_slot, }; #define RT1308_STEREO_RATES SNDRV_PCM_RATE_48000 diff --git a/sound/soc/codecs/rt1308-sdw.h b/sound/soc/codecs/rt1308-sdw.h index c9341e70d6cf..c5ce75666dcc 100644 --- a/sound/soc/codecs/rt1308-sdw.h +++ b/sound/soc/codecs/rt1308-sdw.h @@ -160,6 +160,8 @@ struct rt1308_sdw_priv { struct sdw_bus_params params; bool hw_init; bool first_hw_init; + int rx_mask; + int slots; }; struct sdw_stream_data { diff --git a/sound/soc/codecs/rt5659.c b/sound/soc/codecs/rt5659.c index e66d08398f74..89e0f58512fa 100644 --- a/sound/soc/codecs/rt5659.c +++ b/sound/soc/codecs/rt5659.c @@ -1604,7 +1604,7 @@ static int set_dmic_clk(struct snd_soc_dapm_widget *w, { struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); struct rt5659_priv *rt5659 = snd_soc_component_get_drvdata(component); - int pd, idx = -EINVAL; + int pd, idx; pd = rl6231_get_pre_div(rt5659->regmap, RT5659_ADDA_CLK_1, RT5659_I2S_PD1_SFT); diff --git a/sound/soc/codecs/rt5682-sdw.c b/sound/soc/codecs/rt5682-sdw.c new file mode 100644 index 000000000000..a2d1d3ae1e31 --- /dev/null +++ b/sound/soc/codecs/rt5682-sdw.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// rt5682-sdw.c -- RT5682 ALSA SoC audio component driver +// +// Copyright 2019 Realtek Semiconductor Corp. +// Author: Oder Chiou +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rt5682.h" +#include "rt5682-sdw.h" + +static bool rt5682_sdw_readable_register(struct device *dev, unsigned int reg) +{ + switch (reg) { + case 0x00e0: + case 0x00f0: + case 0x3000: + case 0x3001: + case 0x3004: + case 0x3005: + case 0x3008: + return true; + default: + return false; + } +} + +const struct regmap_config rt5682_sdw_regmap = { + .name = "sdw", + .reg_bits = 32, + .val_bits = 8, + .max_register = RT5682_I2C_MODE, + .readable_reg = rt5682_sdw_readable_register, + .cache_type = REGCACHE_NONE, + .use_single_read = true, + .use_single_write = true, +}; + +static int rt5682_update_status(struct sdw_slave *slave, + enum sdw_slave_status status) +{ + struct rt5682_priv *rt5682 = dev_get_drvdata(&slave->dev); + + /* Update the status */ + rt5682->status = status; + + if (status == SDW_SLAVE_UNATTACHED) + rt5682->hw_init = false; + + /* + * Perform initialization only if slave status is present and + * hw_init flag is false + */ + if (rt5682->hw_init || rt5682->status != SDW_SLAVE_ATTACHED) + return 0; + + /* perform I/O transfers required for Slave initialization */ + return rt5682_io_init(&slave->dev, slave); +} + +static int rt5682_read_prop(struct sdw_slave *slave) +{ + struct sdw_slave_prop *prop = &slave->prop; + int nval, i, num_of_ports = 1; + u32 bit; + unsigned long addr; + struct sdw_dpn_prop *dpn; + + prop->paging_support = false; + + /* first we need to allocate memory for set bits in port lists */ + prop->source_ports = 0x4; /* BITMAP: 00000100 */ + prop->sink_ports = 0x2; /* BITMAP: 00000010 */ + + nval = hweight32(prop->source_ports); + num_of_ports += nval; + prop->src_dpn_prop = devm_kcalloc(&slave->dev, nval, + sizeof(*prop->src_dpn_prop), + GFP_KERNEL); + if (!prop->src_dpn_prop) + return -ENOMEM; + + i = 0; + dpn = prop->src_dpn_prop; + addr = prop->source_ports; + for_each_set_bit(bit, &addr, 32) { + dpn[i].num = bit; + dpn[i].type = SDW_DPN_FULL; + dpn[i].simple_ch_prep_sm = true; + dpn[i].ch_prep_timeout = 10; + i++; + } + + /* do this again for sink now */ + nval = hweight32(prop->sink_ports); + num_of_ports += nval; + prop->sink_dpn_prop = devm_kcalloc(&slave->dev, nval, + sizeof(*prop->sink_dpn_prop), + GFP_KERNEL); + if (!prop->sink_dpn_prop) + return -ENOMEM; + + i = 0; + dpn = prop->sink_dpn_prop; + addr = prop->sink_ports; + for_each_set_bit(bit, &addr, 32) { + dpn[i].num = bit; + dpn[i].type = SDW_DPN_FULL; + dpn[i].simple_ch_prep_sm = true; + dpn[i].ch_prep_timeout = 10; + i++; + } + + /* Allocate port_ready based on num_of_ports */ + slave->port_ready = devm_kcalloc(&slave->dev, num_of_ports, + sizeof(*slave->port_ready), + GFP_KERNEL); + if (!slave->port_ready) + return -ENOMEM; + + /* Initialize completion */ + for (i = 0; i < num_of_ports; i++) + init_completion(&slave->port_ready[i]); + + /* set the timeout values */ + prop->clk_stop_timeout = 20; + + /* wake-up event */ + prop->wake_capable = 1; + + return 0; +} + +/* Bus clock frequency */ +#define RT5682_CLK_FREQ_9600000HZ 9600000 +#define RT5682_CLK_FREQ_12000000HZ 12000000 +#define RT5682_CLK_FREQ_6000000HZ 6000000 +#define RT5682_CLK_FREQ_4800000HZ 4800000 +#define RT5682_CLK_FREQ_2400000HZ 2400000 +#define RT5682_CLK_FREQ_12288000HZ 12288000 + +static int rt5682_clock_config(struct device *dev) +{ + struct rt5682_priv *rt5682 = dev_get_drvdata(dev); + unsigned int clk_freq, value; + + clk_freq = (rt5682->params.curr_dr_freq >> 1); + + switch (clk_freq) { + case RT5682_CLK_FREQ_12000000HZ: + value = 0x0; + break; + case RT5682_CLK_FREQ_6000000HZ: + value = 0x1; + break; + case RT5682_CLK_FREQ_9600000HZ: + value = 0x2; + break; + case RT5682_CLK_FREQ_4800000HZ: + value = 0x3; + break; + case RT5682_CLK_FREQ_2400000HZ: + value = 0x4; + break; + case RT5682_CLK_FREQ_12288000HZ: + value = 0x5; + break; + default: + return -EINVAL; + } + + regmap_write(rt5682->sdw_regmap, 0xe0, value); + regmap_write(rt5682->sdw_regmap, 0xf0, value); + + dev_dbg(dev, "%s complete, clk_freq=%d\n", __func__, clk_freq); + + return 0; +} + +static int rt5682_bus_config(struct sdw_slave *slave, + struct sdw_bus_params *params) +{ + struct rt5682_priv *rt5682 = dev_get_drvdata(&slave->dev); + int ret; + + memcpy(&rt5682->params, params, sizeof(*params)); + + ret = rt5682_clock_config(&slave->dev); + if (ret < 0) + dev_err(&slave->dev, "Invalid clk config"); + + return ret; +} + +static int rt5682_interrupt_callback(struct sdw_slave *slave, + struct sdw_slave_intr_status *status) +{ + struct rt5682_priv *rt5682 = dev_get_drvdata(&slave->dev); + + dev_dbg(&slave->dev, + "%s control_port_stat=%x", __func__, status->control_port); + + if (status->control_port & 0x4) { + mod_delayed_work(system_power_efficient_wq, + &rt5682->jack_detect_work, msecs_to_jiffies(250)); + } + + return 0; +} + +static struct sdw_slave_ops rt5682_slave_ops = { + .read_prop = rt5682_read_prop, + .interrupt_callback = rt5682_interrupt_callback, + .update_status = rt5682_update_status, + .bus_config = rt5682_bus_config, +}; + +static int rt5682_sdw_probe(struct sdw_slave *slave, + const struct sdw_device_id *id) +{ + struct regmap *regmap; + + /* Assign ops */ + slave->ops = &rt5682_slave_ops; + + /* Regmap Initialization */ + regmap = devm_regmap_init_sdw(slave, &rt5682_sdw_regmap); + if (IS_ERR(regmap)) + return -EINVAL; + + rt5682_sdw_init(&slave->dev, regmap, slave); + + return 0; +} + +static int rt5682_sdw_remove(struct sdw_slave *slave) +{ + struct rt5682_priv *rt5682 = dev_get_drvdata(&slave->dev); + + if (rt5682 && rt5682->hw_init) + cancel_delayed_work(&rt5682->jack_detect_work); + + return 0; +} + +static const struct sdw_device_id rt5682_id[] = { + SDW_SLAVE_ENTRY(0x025d, 0x5682, 0), + {}, +}; +MODULE_DEVICE_TABLE(sdw, rt5682_id); + +static int __maybe_unused rt5682_dev_suspend(struct device *dev) +{ + struct rt5682_priv *rt5682 = dev_get_drvdata(dev); + + if (!rt5682->hw_init) + return 0; + + regcache_cache_only(rt5682->regmap, true); + regcache_mark_dirty(rt5682->regmap); + + return 0; +} + +static int __maybe_unused rt5682_dev_resume(struct device *dev) +{ + struct sdw_slave *slave = dev_to_sdw_dev(dev); + struct rt5682_priv *rt5682 = dev_get_drvdata(dev); + unsigned long time; + + if (!rt5682->hw_init) + return 0; + + if (!slave->unattach_request) + goto regmap_sync; + + time = wait_for_completion_timeout(&slave->initialization_complete, + msecs_to_jiffies(RT5682_PROBE_TIMEOUT)); + if (!time) { + dev_err(&slave->dev, "Initialization not complete, timed out\n"); + return -ETIMEDOUT; + } + +regmap_sync: + slave->unattach_request = 0; + regcache_cache_only(rt5682->regmap, false); + regcache_sync(rt5682->regmap); + + return 0; +} + +static const struct dev_pm_ops rt5682_pm = { + SET_SYSTEM_SLEEP_PM_OPS(rt5682_dev_suspend, rt5682_dev_resume) + SET_RUNTIME_PM_OPS(rt5682_dev_suspend, rt5682_dev_resume, NULL) +}; + +static struct sdw_driver rt5682_sdw_driver = { + .driver = { + .name = "rt5682", + .owner = THIS_MODULE, + .pm = &rt5682_pm, + }, + .probe = rt5682_sdw_probe, + .remove = rt5682_sdw_remove, + .ops = &rt5682_slave_ops, + .id_table = rt5682_id, +}; +module_sdw_driver(rt5682_sdw_driver); + +MODULE_DESCRIPTION("ASoC RT5682 driver SDW"); +MODULE_AUTHOR("Oder Chiou "); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/codecs/rt5682-sdw.h b/sound/soc/codecs/rt5682-sdw.h new file mode 100644 index 000000000000..76e6f607066e --- /dev/null +++ b/sound/soc/codecs/rt5682-sdw.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * + * rt5682-sdw.h -- RT5682 SDW ALSA SoC audio driver + * + * Copyright 2019 Realtek Semiconductor Corp. + * Author: Oder Chiou + */ + +#ifndef __RT5682_SDW_H__ +#define __RT5682_SDW_H__ + +#define RT5682_SDW_ADDR_L 0x3000 +#define RT5682_SDW_ADDR_H 0x3001 +#define RT5682_SDW_DATA_L 0x3004 +#define RT5682_SDW_DATA_H 0x3005 +#define RT5682_SDW_CMD 0x3008 + +#define RT5682_PROBE_TIMEOUT 2000 + +#endif /* __RT5682_SDW_H__ */ diff --git a/sound/soc/codecs/rt5682.c b/sound/soc/codecs/rt5682.c index ae6f6121bc1b..c9268a230daa 100644 --- a/sound/soc/codecs/rt5682.c +++ b/sound/soc/codecs/rt5682.c @@ -11,13 +11,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include @@ -31,8 +31,7 @@ #include "rl6231.h" #include "rt5682.h" - -#define RT5682_NUM_SUPPLIES 3 +#include "rt5682-sdw.h" static const char *rt5682_supply_names[RT5682_NUM_SUPPLIES] = { "AVDD", @@ -45,35 +44,15 @@ static const struct rt5682_platform_data i2s_default_platform_data = { .dmic1_clk_pin = RT5682_DMIC1_CLK_GPIO3, .jd_src = RT5682_JD1, .btndet_delay = 16, -}; - -struct rt5682_priv { - struct snd_soc_component *component; - struct rt5682_platform_data pdata; - struct regmap *regmap; - struct snd_soc_jack *hs_jack; - struct regulator_bulk_data supplies[RT5682_NUM_SUPPLIES]; - struct delayed_work jack_detect_work; - struct delayed_work jd_check_work; - struct mutex calibrate_mutex; - - int sysclk; - int sysclk_src; - int lrck[RT5682_AIFS]; - int bclk[RT5682_AIFS]; - int master[RT5682_AIFS]; - - int pll_src; - int pll_in; - int pll_out; - - int jack_type; + .dai_clk_names[RT5682_DAI_WCLK_IDX] = "rt5682-dai-wclk", + .dai_clk_names[RT5682_DAI_BCLK_IDX] = "rt5682-dai-bclk", }; static const struct reg_sequence patch_list[] = { {RT5682_HP_IMP_SENS_CTRL_19, 0x1000}, {RT5682_DAC_ADC_DIG_VOL1, 0xa020}, {RT5682_I2C_CTRL, 0x000f}, + {RT5682_PLL2_INTERNAL, 0x8266}, }; static const struct reg_default rt5682_reg[] = { @@ -221,7 +200,7 @@ static const struct reg_default rt5682_reg[] = { {0x0148, 0x0000}, {0x0149, 0x0000}, {0x0150, 0x79a1}, - {0x0151, 0x0000}, + {0x0156, 0xaaaa}, {0x0160, 0x4ec0}, {0x0161, 0x0080}, {0x0162, 0x0200}, @@ -805,10 +784,27 @@ static const struct snd_kcontrol_new rt5682_if1_45_adc_swap_mux = static const struct snd_kcontrol_new rt5682_if1_67_adc_swap_mux = SOC_DAPM_ENUM("IF1 67 ADC Swap Mux", rt5682_if1_67_adc_enum); -static void rt5682_reset(struct regmap *regmap) +static const char * const rt5682_dac_select[] = { + "IF1", "SOUND" +}; + +static SOC_ENUM_SINGLE_DECL(rt5682_dacl_enum, + RT5682_AD_DA_MIXER, RT5682_DAC1_L_SEL_SFT, rt5682_dac_select); + +static const struct snd_kcontrol_new rt5682_dac_l_mux = + SOC_DAPM_ENUM("DAC L Mux", rt5682_dacl_enum); + +static SOC_ENUM_SINGLE_DECL(rt5682_dacr_enum, + RT5682_AD_DA_MIXER, RT5682_DAC1_R_SEL_SFT, rt5682_dac_select); + +static const struct snd_kcontrol_new rt5682_dac_r_mux = + SOC_DAPM_ENUM("DAC R Mux", rt5682_dacr_enum); + +static void rt5682_reset(struct rt5682_priv *rt5682) { - regmap_write(regmap, RT5682_RESET, 0); - regmap_write(regmap, RT5682_I2C_MODE, 1); + regmap_write(rt5682->regmap, RT5682_RESET, 0); + if (!rt5682->is_sdw) + regmap_write(rt5682->regmap, RT5682_I2C_MODE, 1); } /** * rt5682_sel_asrc_clk_src - select ASRC clock source for a set of filters @@ -871,6 +867,8 @@ static int rt5682_button_detect(struct snd_soc_component *component) static void rt5682_enable_push_button_irq(struct snd_soc_component *component, bool enable) { + struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); + if (enable) { snd_soc_component_update_bits(component, RT5682_SAR_IL_CMD_1, RT5682_SAR_BUTT_DET_MASK, RT5682_SAR_BUTT_DET_EN); @@ -880,8 +878,15 @@ static void rt5682_enable_push_button_irq(struct snd_soc_component *component, snd_soc_component_update_bits(component, RT5682_4BTN_IL_CMD_2, RT5682_4BTN_IL_MASK | RT5682_4BTN_IL_RST_MASK, RT5682_4BTN_IL_EN | RT5682_4BTN_IL_NOR); - snd_soc_component_update_bits(component, RT5682_IRQ_CTRL_3, - RT5682_IL_IRQ_MASK, RT5682_IL_IRQ_EN); + if (rt5682->is_sdw) + snd_soc_component_update_bits(component, + RT5682_IRQ_CTRL_3, + RT5682_IL_IRQ_MASK | RT5682_IL_IRQ_TYPE_MASK, + RT5682_IL_IRQ_EN | RT5682_IL_IRQ_PUL); + else + snd_soc_component_update_bits(component, + RT5682_IRQ_CTRL_3, RT5682_IL_IRQ_MASK, + RT5682_IL_IRQ_EN); } else { snd_soc_component_update_bits(component, RT5682_IRQ_CTRL_3, RT5682_IL_IRQ_MASK, RT5682_IL_IRQ_DIS); @@ -909,6 +914,7 @@ static int rt5682_headset_detect(struct snd_soc_component *component, int jack_insert) { struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); + struct snd_soc_dapm_context *dapm = &component->dapm; unsigned int val, count; if (jack_insert) { @@ -917,10 +923,10 @@ static int rt5682_headset_detect(struct snd_soc_component *component, RT5682_PWR_VREF2 | RT5682_PWR_MB, RT5682_PWR_VREF2 | RT5682_PWR_MB); snd_soc_component_update_bits(component, - RT5682_PWR_ANLG_1, RT5682_PWR_FV2, 0); + RT5682_PWR_ANLG_1, RT5682_PWR_FV2, 0); usleep_range(15000, 20000); snd_soc_component_update_bits(component, - RT5682_PWR_ANLG_1, RT5682_PWR_FV2, RT5682_PWR_FV2); + RT5682_PWR_ANLG_1, RT5682_PWR_FV2, RT5682_PWR_FV2); snd_soc_component_update_bits(component, RT5682_PWR_ANLG_3, RT5682_PWR_CBJ, RT5682_PWR_CBJ); @@ -951,8 +957,13 @@ static int rt5682_headset_detect(struct snd_soc_component *component, rt5682_enable_push_button_irq(component, false); snd_soc_component_update_bits(component, RT5682_CBJ_CTRL_1, RT5682_TRIG_JD_MASK, RT5682_TRIG_JD_LOW); - snd_soc_component_update_bits(component, RT5682_PWR_ANLG_1, - RT5682_PWR_VREF2 | RT5682_PWR_MB, 0); + if (snd_soc_dapm_get_pin_status(dapm, "MICBIAS")) + snd_soc_component_update_bits(component, + RT5682_PWR_ANLG_1, RT5682_PWR_VREF2, 0); + else + snd_soc_component_update_bits(component, + RT5682_PWR_ANLG_1, + RT5682_PWR_VREF2 | RT5682_PWR_MB, 0); snd_soc_component_update_bits(component, RT5682_PWR_ANLG_3, RT5682_PWR_CBJ, 0); @@ -999,62 +1010,69 @@ static int rt5682_set_jack_detect(struct snd_soc_component *component, rt5682->hs_jack = hs_jack; - if (!hs_jack) { - regmap_update_bits(rt5682->regmap, RT5682_IRQ_CTRL_2, - RT5682_JD1_EN_MASK, RT5682_JD1_DIS); - regmap_update_bits(rt5682->regmap, RT5682_RC_CLK_CTRL, - RT5682_POW_JDH | RT5682_POW_JDL, 0); - cancel_delayed_work_sync(&rt5682->jack_detect_work); - return 0; - } + if (!rt5682->is_sdw) { + if (!hs_jack) { + regmap_update_bits(rt5682->regmap, RT5682_IRQ_CTRL_2, + RT5682_JD1_EN_MASK, RT5682_JD1_DIS); + regmap_update_bits(rt5682->regmap, RT5682_RC_CLK_CTRL, + RT5682_POW_JDH | RT5682_POW_JDL, 0); + cancel_delayed_work_sync(&rt5682->jack_detect_work); + return 0; + } - switch (rt5682->pdata.jd_src) { - case RT5682_JD1: - snd_soc_component_update_bits(component, RT5682_CBJ_CTRL_2, - RT5682_EXT_JD_SRC, RT5682_EXT_JD_SRC_MANUAL); - snd_soc_component_write(component, RT5682_CBJ_CTRL_1, 0xd042); - snd_soc_component_update_bits(component, RT5682_CBJ_CTRL_3, - RT5682_CBJ_IN_BUF_EN, RT5682_CBJ_IN_BUF_EN); - snd_soc_component_update_bits(component, RT5682_SAR_IL_CMD_1, - RT5682_SAR_POW_MASK, RT5682_SAR_POW_EN); - regmap_update_bits(rt5682->regmap, RT5682_GPIO_CTRL_1, - RT5682_GP1_PIN_MASK, RT5682_GP1_PIN_IRQ); - regmap_update_bits(rt5682->regmap, RT5682_RC_CLK_CTRL, + switch (rt5682->pdata.jd_src) { + case RT5682_JD1: + snd_soc_component_update_bits(component, + RT5682_CBJ_CTRL_2, RT5682_EXT_JD_SRC, + RT5682_EXT_JD_SRC_MANUAL); + snd_soc_component_write(component, RT5682_CBJ_CTRL_1, + 0xd042); + snd_soc_component_update_bits(component, + RT5682_CBJ_CTRL_3, RT5682_CBJ_IN_BUF_EN, + RT5682_CBJ_IN_BUF_EN); + snd_soc_component_update_bits(component, + RT5682_SAR_IL_CMD_1, RT5682_SAR_POW_MASK, + RT5682_SAR_POW_EN); + regmap_update_bits(rt5682->regmap, RT5682_GPIO_CTRL_1, + RT5682_GP1_PIN_MASK, RT5682_GP1_PIN_IRQ); + regmap_update_bits(rt5682->regmap, RT5682_RC_CLK_CTRL, RT5682_POW_IRQ | RT5682_POW_JDH | RT5682_POW_ANA, RT5682_POW_IRQ | RT5682_POW_JDH | RT5682_POW_ANA); - regmap_update_bits(rt5682->regmap, RT5682_PWR_ANLG_2, - RT5682_PWR_JDH | RT5682_PWR_JDL, - RT5682_PWR_JDH | RT5682_PWR_JDL); - regmap_update_bits(rt5682->regmap, RT5682_IRQ_CTRL_2, - RT5682_JD1_EN_MASK | RT5682_JD1_POL_MASK, - RT5682_JD1_EN | RT5682_JD1_POL_NOR); - regmap_update_bits(rt5682->regmap, RT5682_4BTN_IL_CMD_4, - 0x7f7f, (rt5682->pdata.btndet_delay << 8 | - rt5682->pdata.btndet_delay)); - regmap_update_bits(rt5682->regmap, RT5682_4BTN_IL_CMD_5, - 0x7f7f, (rt5682->pdata.btndet_delay << 8 | - rt5682->pdata.btndet_delay)); - regmap_update_bits(rt5682->regmap, RT5682_4BTN_IL_CMD_6, - 0x7f7f, (rt5682->pdata.btndet_delay << 8 | - rt5682->pdata.btndet_delay)); - regmap_update_bits(rt5682->regmap, RT5682_4BTN_IL_CMD_7, - 0x7f7f, (rt5682->pdata.btndet_delay << 8 | - rt5682->pdata.btndet_delay)); - mod_delayed_work(system_power_efficient_wq, - &rt5682->jack_detect_work, msecs_to_jiffies(250)); - break; + regmap_update_bits(rt5682->regmap, RT5682_PWR_ANLG_2, + RT5682_PWR_JDH | RT5682_PWR_JDL, + RT5682_PWR_JDH | RT5682_PWR_JDL); + regmap_update_bits(rt5682->regmap, RT5682_IRQ_CTRL_2, + RT5682_JD1_EN_MASK | RT5682_JD1_POL_MASK, + RT5682_JD1_EN | RT5682_JD1_POL_NOR); + regmap_update_bits(rt5682->regmap, RT5682_4BTN_IL_CMD_4, + 0x7f7f, (rt5682->pdata.btndet_delay << 8 | + rt5682->pdata.btndet_delay)); + regmap_update_bits(rt5682->regmap, RT5682_4BTN_IL_CMD_5, + 0x7f7f, (rt5682->pdata.btndet_delay << 8 | + rt5682->pdata.btndet_delay)); + regmap_update_bits(rt5682->regmap, RT5682_4BTN_IL_CMD_6, + 0x7f7f, (rt5682->pdata.btndet_delay << 8 | + rt5682->pdata.btndet_delay)); + regmap_update_bits(rt5682->regmap, RT5682_4BTN_IL_CMD_7, + 0x7f7f, (rt5682->pdata.btndet_delay << 8 | + rt5682->pdata.btndet_delay)); + mod_delayed_work(system_power_efficient_wq, + &rt5682->jack_detect_work, + msecs_to_jiffies(250)); + break; - case RT5682_JD_NULL: - regmap_update_bits(rt5682->regmap, RT5682_IRQ_CTRL_2, - RT5682_JD1_EN_MASK, RT5682_JD1_DIS); - regmap_update_bits(rt5682->regmap, RT5682_RC_CLK_CTRL, - RT5682_POW_JDH | RT5682_POW_JDL, 0); - break; + case RT5682_JD_NULL: + regmap_update_bits(rt5682->regmap, RT5682_IRQ_CTRL_2, + RT5682_JD1_EN_MASK, RT5682_JD1_DIS); + regmap_update_bits(rt5682->regmap, RT5682_RC_CLK_CTRL, + RT5682_POW_JDH | RT5682_POW_JDL, 0); + break; - default: - dev_warn(component->dev, "Wrong JD source\n"); - break; + default: + dev_warn(component->dev, "Wrong JD source\n"); + break; + } } return 0; @@ -1134,11 +1152,13 @@ static void rt5682_jack_detect_handler(struct work_struct *work) SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3); - if (rt5682->jack_type & (SND_JACK_BTN_0 | SND_JACK_BTN_1 | - SND_JACK_BTN_2 | SND_JACK_BTN_3)) - schedule_delayed_work(&rt5682->jd_check_work, 0); - else - cancel_delayed_work_sync(&rt5682->jd_check_work); + if (!rt5682->is_sdw) { + if (rt5682->jack_type & (SND_JACK_BTN_0 | SND_JACK_BTN_1 | + SND_JACK_BTN_2 | SND_JACK_BTN_3)) + schedule_delayed_work(&rt5682->jd_check_work, 0); + else + cancel_delayed_work_sync(&rt5682->jd_check_work); + } mutex_unlock(&rt5682->calibrate_mutex); } @@ -1146,7 +1166,7 @@ static void rt5682_jack_detect_handler(struct work_struct *work) static const struct snd_kcontrol_new rt5682_snd_controls[] = { /* DAC Digital Volume */ SOC_DOUBLE_TLV("DAC1 Playback Volume", RT5682_DAC1_DIG_VOL, - RT5682_L_VOL_SFT + 1, RT5682_R_VOL_SFT + 1, 86, 0, dac_vol_tlv), + RT5682_L_VOL_SFT + 1, RT5682_R_VOL_SFT + 1, 87, 0, dac_vol_tlv), /* IN Boost Volume */ SOC_SINGLE_TLV("CBJ Boost Volume", RT5682_CBJ_BST_CTRL, @@ -1177,11 +1197,11 @@ static int rt5682_div_sel(struct rt5682_priv *rt5682, } for (i = 0; i < size - 1; i++) { - pr_info("div[%d]=%d\n", i, div[i]); + dev_dbg(rt5682->component->dev, "div[%d]=%d\n", i, div[i]); if (target * div[i] == rt5682->sysclk) return i; if (target * div[i + 1] > rt5682->sysclk) { - pr_err("can't find div for sysclk %d\n", + dev_dbg(rt5682->component->dev, "can't find div for sysclk %d\n", rt5682->sysclk); return i; } @@ -1211,10 +1231,13 @@ static int set_dmic_clk(struct snd_soc_dapm_widget *w, struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); - int idx = -EINVAL; + int idx = -EINVAL, dmic_clk_rate = 3072000; static const int div[] = {2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128}; - idx = rt5682_div_sel(rt5682, 1500000, div, ARRAY_SIZE(div)); + if (rt5682->pdata.dmic_clk_rate) + dmic_clk_rate = rt5682->pdata.dmic_clk_rate; + + idx = rt5682_div_sel(rt5682, dmic_clk_rate, div, ARRAY_SIZE(div)); snd_soc_component_update_bits(component, RT5682_DMIC_CTRL_1, RT5682_DMIC_CLK_MASK, idx << RT5682_DMIC_CLK_SFT); @@ -1232,6 +1255,9 @@ static int set_filter_clk(struct snd_soc_dapm_widget *w, static const int div_f[] = {1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48}; static const int div_o[] = {1, 2, 4, 6, 8, 12, 16, 24, 32, 48}; + if (rt5682->is_sdw) + return 0; + val = snd_soc_component_read32(component, RT5682_GPIO_CTRL_1) & RT5682_GP4_PIN_MASK; if (w->shift == RT5682_PWR_ADC_S1F_BIT && @@ -1278,6 +1304,21 @@ static int is_sys_clk_from_pll1(struct snd_soc_dapm_widget *w, return 0; } +static int is_sys_clk_from_pll2(struct snd_soc_dapm_widget *w, + struct snd_soc_dapm_widget *sink) +{ + unsigned int val; + struct snd_soc_component *component = + snd_soc_dapm_to_component(w->dapm); + + val = snd_soc_component_read32(component, RT5682_GLB_CLK); + val &= RT5682_SCLK_SRC_MASK; + if (val == RT5682_SCLK_SRC_PLL2) + return 1; + else + return 0; +} + static int is_using_asrc(struct snd_soc_dapm_widget *w, struct snd_soc_dapm_widget *sink) { @@ -1503,10 +1544,18 @@ static int rt5682_hp_event(struct snd_soc_dapm_widget *w, static int set_dmic_power(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol, int event) { + struct snd_soc_component *component = + snd_soc_dapm_to_component(w->dapm); + struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); + unsigned int delay = 50; + + if (rt5682->pdata.dmic_delay) + delay = rt5682->pdata.dmic_delay; + switch (event) { case SND_SOC_DAPM_POST_PMU: /*Add delay to avoid pop noise*/ - msleep(150); + msleep(delay); break; default: @@ -1516,7 +1565,7 @@ static int set_dmic_power(struct snd_soc_dapm_widget *w, return 0; } -static int rt5655_set_verf(struct snd_soc_dapm_widget *w, +static int rt5682_set_verf(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol, int event) { struct snd_soc_component *component = @@ -1592,9 +1641,12 @@ static const struct snd_soc_dapm_widget rt5682_dapm_widgets[] = { SND_SOC_DAPM_SUPPLY("PLL2B", RT5682_PWR_ANLG_3, RT5682_PWR_PLL2B_BIT, 0, NULL, 0), SND_SOC_DAPM_SUPPLY("PLL2F", RT5682_PWR_ANLG_3, RT5682_PWR_PLL2F_BIT, - 0, NULL, 0), + 0, set_filter_clk, SND_SOC_DAPM_PRE_PMU), SND_SOC_DAPM_SUPPLY("Vref1", RT5682_PWR_ANLG_1, RT5682_PWR_VREF1_BIT, 0, - rt5655_set_verf, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU), + rt5682_set_verf, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU), + SND_SOC_DAPM_SUPPLY("Vref2", RT5682_PWR_ANLG_1, RT5682_PWR_VREF2_BIT, 0, + NULL, 0), + SND_SOC_DAPM_SUPPLY("MICBIAS", SND_SOC_NOPM, 0, 0, NULL, 0), /* ASRC */ SND_SOC_DAPM_SUPPLY_S("DAC STO1 ASRC", 1, RT5682_PLL_TRACK_1, @@ -1686,6 +1738,8 @@ static const struct snd_soc_dapm_widget rt5682_dapm_widgets[] = { SND_SOC_DAPM_PGA("IF1 DAC1", SND_SOC_NOPM, 0, 0, NULL, 0), SND_SOC_DAPM_PGA("IF1 DAC1 L", SND_SOC_NOPM, 0, 0, NULL, 0), SND_SOC_DAPM_PGA("IF1 DAC1 R", SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_PGA("SOUND DAC L", SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_PGA("SOUND DAC R", SND_SOC_NOPM, 0, 0, NULL, 0), /* Digital Interface Select */ SND_SOC_DAPM_MUX("IF1 01 ADC Swap Mux", SND_SOC_NOPM, 0, 0, @@ -1702,12 +1756,19 @@ static const struct snd_soc_dapm_widget rt5682_dapm_widgets[] = { SND_SOC_DAPM_MUX("ADCDAT Mux", SND_SOC_NOPM, 0, 0, &rt5682_adcdat_pin_ctrl), + SND_SOC_DAPM_MUX("DAC L Mux", SND_SOC_NOPM, 0, 0, + &rt5682_dac_l_mux), + SND_SOC_DAPM_MUX("DAC R Mux", SND_SOC_NOPM, 0, 0, + &rt5682_dac_r_mux), + /* Audio Interface */ SND_SOC_DAPM_AIF_OUT("AIF1TX", "AIF1 Capture", 0, RT5682_I2S1_SDP, RT5682_SEL_ADCDAT_SFT, 1), SND_SOC_DAPM_AIF_OUT("AIF2TX", "AIF2 Capture", 0, RT5682_I2S2_SDP, RT5682_I2S2_PIN_CFG_SFT, 1), SND_SOC_DAPM_AIF_IN("AIF1RX", "AIF1 Playback", 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_IN("SDWRX", "SDW Playback", 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_OUT("SDWTX", "SDW Capture", 0, SND_SOC_NOPM, 0, 0), /* Output Side */ /* DAC mixer before sound effect */ @@ -1776,7 +1837,11 @@ static const struct snd_soc_dapm_widget rt5682_dapm_widgets[] = { static const struct snd_soc_dapm_route rt5682_dapm_routes[] = { /*PLL*/ {"ADC Stereo1 Filter", NULL, "PLL1", is_sys_clk_from_pll1}, + {"ADC Stereo1 Filter", NULL, "PLL2B", is_sys_clk_from_pll2}, + {"ADC Stereo1 Filter", NULL, "PLL2F", is_sys_clk_from_pll2}, {"DAC Stereo1 Filter", NULL, "PLL1", is_sys_clk_from_pll1}, + {"DAC Stereo1 Filter", NULL, "PLL2B", is_sys_clk_from_pll2}, + {"DAC Stereo1 Filter", NULL, "PLL2F", is_sys_clk_from_pll2}, /*ASRC*/ {"ADC Stereo1 Filter", NULL, "ADC STO1 ASRC", is_using_asrc}, @@ -1860,8 +1925,8 @@ static const struct snd_soc_dapm_route rt5682_dapm_routes[] = { {"IF1_ADC Mux", "Slot 2", "IF1 23 ADC Swap Mux"}, {"IF1_ADC Mux", "Slot 4", "IF1 45 ADC Swap Mux"}, {"IF1_ADC Mux", "Slot 6", "IF1 67 ADC Swap Mux"}, - {"IF1_ADC Mux", NULL, "I2S1"}, {"ADCDAT Mux", "ADCDAT1", "IF1_ADC Mux"}, + {"AIF1TX", NULL, "I2S1"}, {"AIF1TX", NULL, "ADCDAT Mux"}, {"IF2 ADC Swap Mux", "L/R", "Stereo1 ADC MIX"}, {"IF2 ADC Swap Mux", "R/L", "Stereo1 ADC MIX"}, @@ -1870,6 +1935,10 @@ static const struct snd_soc_dapm_route rt5682_dapm_routes[] = { {"ADCDAT Mux", "ADCDAT2", "IF2 ADC Swap Mux"}, {"AIF2TX", NULL, "ADCDAT Mux"}, + {"SDWTX", NULL, "PLL2B"}, + {"SDWTX", NULL, "PLL2F"}, + {"SDWTX", NULL, "ADCDAT Mux"}, + {"IF1 DAC1 L", NULL, "AIF1RX"}, {"IF1 DAC1 L", NULL, "I2S1"}, {"IF1 DAC1 L", NULL, "DAC Stereo1 Filter"}, @@ -1877,10 +1946,24 @@ static const struct snd_soc_dapm_route rt5682_dapm_routes[] = { {"IF1 DAC1 R", NULL, "I2S1"}, {"IF1 DAC1 R", NULL, "DAC Stereo1 Filter"}, + {"SOUND DAC L", NULL, "SDWRX"}, + {"SOUND DAC L", NULL, "DAC Stereo1 Filter"}, + {"SOUND DAC L", NULL, "PLL2B"}, + {"SOUND DAC L", NULL, "PLL2F"}, + {"SOUND DAC R", NULL, "SDWRX"}, + {"SOUND DAC R", NULL, "DAC Stereo1 Filter"}, + {"SOUND DAC R", NULL, "PLL2B"}, + {"SOUND DAC R", NULL, "PLL2F"}, + + {"DAC L Mux", "IF1", "IF1 DAC1 L"}, + {"DAC L Mux", "SOUND", "SOUND DAC L"}, + {"DAC R Mux", "IF1", "IF1 DAC1 R"}, + {"DAC R Mux", "SOUND", "SOUND DAC R"}, + {"DAC1 MIXL", "Stereo ADC Switch", "Stereo1 ADC MIXL"}, - {"DAC1 MIXL", "DAC1 Switch", "IF1 DAC1 L"}, + {"DAC1 MIXL", "DAC1 Switch", "DAC L Mux"}, {"DAC1 MIXR", "Stereo ADC Switch", "Stereo1 ADC MIXR"}, - {"DAC1 MIXR", "DAC1 Switch", "IF1 DAC1 R"}, + {"DAC1 MIXR", "DAC1 Switch", "DAC R Mux"}, {"Stereo1 DAC MIXL", "DAC L1 Switch", "DAC1 MIXL"}, {"Stereo1 DAC MIXL", "DAC R1 Switch", "DAC1 MIXR"}, @@ -2033,8 +2116,10 @@ static int rt5682_hw_params(struct snd_pcm_substream *substream, RT5682_I2S1_DL_MASK, len_1); if (rt5682->master[RT5682_AIF1]) { snd_soc_component_update_bits(component, - RT5682_ADDA_CLK_1, RT5682_I2S_M_DIV_MASK, - pre_div << RT5682_I2S_M_DIV_SFT); + RT5682_ADDA_CLK_1, RT5682_I2S_M_DIV_MASK | + RT5682_I2S_CLK_SRC_MASK, + pre_div << RT5682_I2S_M_DIV_SFT | + (rt5682->sysclk_src) << RT5682_I2S_CLK_SRC_SFT); } if (params_channels(params) == 1) /* mono mode */ snd_soc_component_update_bits(component, @@ -2207,61 +2292,157 @@ static int rt5682_set_component_pll(struct snd_soc_component *component, unsigned int freq_out) { struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); - struct rl6231_pll_code pll_code; + struct rl6231_pll_code pll_code, pll2f_code, pll2b_code; + unsigned int pll2_fout1; int ret; - if (source == rt5682->pll_src && freq_in == rt5682->pll_in && - freq_out == rt5682->pll_out) + if (source == rt5682->pll_src[pll_id] && + freq_in == rt5682->pll_in[pll_id] && + freq_out == rt5682->pll_out[pll_id]) return 0; if (!freq_in || !freq_out) { dev_dbg(component->dev, "PLL disabled\n"); - rt5682->pll_in = 0; - rt5682->pll_out = 0; + rt5682->pll_in[pll_id] = 0; + rt5682->pll_out[pll_id] = 0; snd_soc_component_update_bits(component, RT5682_GLB_CLK, RT5682_SCLK_SRC_MASK, RT5682_SCLK_SRC_MCLK); return 0; } - switch (source) { - case RT5682_PLL1_S_MCLK: - snd_soc_component_update_bits(component, RT5682_GLB_CLK, - RT5682_PLL1_SRC_MASK, RT5682_PLL1_SRC_MCLK); - break; - case RT5682_PLL1_S_BCLK1: - snd_soc_component_update_bits(component, RT5682_GLB_CLK, - RT5682_PLL1_SRC_MASK, RT5682_PLL1_SRC_BCLK1); - break; - default: - dev_err(component->dev, "Unknown PLL Source %d\n", source); - return -EINVAL; + if (pll_id == RT5682_PLL2) { + switch (source) { + case RT5682_PLL2_S_MCLK: + snd_soc_component_update_bits(component, + RT5682_GLB_CLK, RT5682_PLL2_SRC_MASK, + RT5682_PLL2_SRC_MCLK); + break; + default: + dev_err(component->dev, "Unknown PLL2 Source %d\n", + source); + return -EINVAL; + } + + /** + * PLL2 concatenates 2 PLL units. + * We suggest the Fout of the front PLL is 3.84MHz. + */ + pll2_fout1 = 3840000; + ret = rl6231_pll_calc(freq_in, pll2_fout1, &pll2f_code); + if (ret < 0) { + dev_err(component->dev, "Unsupport input clock %d\n", + freq_in); + return ret; + } + dev_dbg(component->dev, "PLL2F: fin=%d fout=%d bypass=%d m=%d n=%d k=%d\n", + freq_in, pll2_fout1, + pll2f_code.m_bp, + (pll2f_code.m_bp ? 0 : pll2f_code.m_code), + pll2f_code.n_code, pll2f_code.k_code); + + ret = rl6231_pll_calc(pll2_fout1, freq_out, &pll2b_code); + if (ret < 0) { + dev_err(component->dev, "Unsupport input clock %d\n", + pll2_fout1); + return ret; + } + dev_dbg(component->dev, "PLL2B: fin=%d fout=%d bypass=%d m=%d n=%d k=%d\n", + pll2_fout1, freq_out, + pll2b_code.m_bp, + (pll2b_code.m_bp ? 0 : pll2b_code.m_code), + pll2b_code.n_code, pll2b_code.k_code); + + snd_soc_component_write(component, RT5682_PLL2_CTRL_1, + pll2f_code.k_code << RT5682_PLL2F_K_SFT | + pll2b_code.k_code << RT5682_PLL2B_K_SFT | + pll2b_code.m_code); + snd_soc_component_write(component, RT5682_PLL2_CTRL_2, + pll2f_code.m_code << RT5682_PLL2F_M_SFT | + pll2b_code.n_code); + snd_soc_component_write(component, RT5682_PLL2_CTRL_3, + pll2f_code.n_code << RT5682_PLL2F_N_SFT); + snd_soc_component_update_bits(component, RT5682_PLL2_CTRL_4, + RT5682_PLL2B_M_BP_MASK | RT5682_PLL2F_M_BP_MASK | 0xf, + (pll2b_code.m_bp ? 1 : 0) << RT5682_PLL2B_M_BP_SFT | + (pll2f_code.m_bp ? 1 : 0) << RT5682_PLL2F_M_BP_SFT | + 0xf); + } else { + switch (source) { + case RT5682_PLL1_S_MCLK: + snd_soc_component_update_bits(component, + RT5682_GLB_CLK, RT5682_PLL1_SRC_MASK, + RT5682_PLL1_SRC_MCLK); + break; + case RT5682_PLL1_S_BCLK1: + snd_soc_component_update_bits(component, + RT5682_GLB_CLK, RT5682_PLL1_SRC_MASK, + RT5682_PLL1_SRC_BCLK1); + break; + default: + dev_err(component->dev, "Unknown PLL1 Source %d\n", + source); + return -EINVAL; + } + + ret = rl6231_pll_calc(freq_in, freq_out, &pll_code); + if (ret < 0) { + dev_err(component->dev, "Unsupport input clock %d\n", + freq_in); + return ret; + } + + dev_dbg(component->dev, "bypass=%d m=%d n=%d k=%d\n", + pll_code.m_bp, (pll_code.m_bp ? 0 : pll_code.m_code), + pll_code.n_code, pll_code.k_code); + + snd_soc_component_write(component, RT5682_PLL_CTRL_1, + pll_code.n_code << RT5682_PLL_N_SFT | pll_code.k_code); + snd_soc_component_write(component, RT5682_PLL_CTRL_2, + (pll_code.m_bp ? 0 : pll_code.m_code) << RT5682_PLL_M_SFT | + pll_code.m_bp << RT5682_PLL_M_BP_SFT | RT5682_PLL_RST); } - ret = rl6231_pll_calc(freq_in, freq_out, &pll_code); - if (ret < 0) { - dev_err(component->dev, "Unsupport input clock %d\n", freq_in); - return ret; - } - - dev_dbg(component->dev, "bypass=%d m=%d n=%d k=%d\n", - pll_code.m_bp, (pll_code.m_bp ? 0 : pll_code.m_code), - pll_code.n_code, pll_code.k_code); - - snd_soc_component_write(component, RT5682_PLL_CTRL_1, - pll_code.n_code << RT5682_PLL_N_SFT | pll_code.k_code); - snd_soc_component_write(component, RT5682_PLL_CTRL_2, - (pll_code.m_bp ? 0 : pll_code.m_code) << RT5682_PLL_M_SFT | - pll_code.m_bp << RT5682_PLL_M_BP_SFT | RT5682_PLL_RST); - - rt5682->pll_in = freq_in; - rt5682->pll_out = freq_out; - rt5682->pll_src = source; + rt5682->pll_in[pll_id] = freq_in; + rt5682->pll_out[pll_id] = freq_out; + rt5682->pll_src[pll_id] = source; return 0; } -static int rt5682_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) +static int rt5682_set_bclk1_ratio(struct snd_soc_dai *dai, unsigned int ratio) +{ + struct snd_soc_component *component = dai->component; + struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); + + rt5682->bclk[dai->id] = ratio; + + switch (ratio) { + case 256: + snd_soc_component_update_bits(component, RT5682_TDM_TCON_CTRL, + RT5682_TDM_BCLK_MS1_MASK, RT5682_TDM_BCLK_MS1_256); + break; + case 128: + snd_soc_component_update_bits(component, RT5682_TDM_TCON_CTRL, + RT5682_TDM_BCLK_MS1_MASK, RT5682_TDM_BCLK_MS1_128); + break; + case 64: + snd_soc_component_update_bits(component, RT5682_TDM_TCON_CTRL, + RT5682_TDM_BCLK_MS1_MASK, RT5682_TDM_BCLK_MS1_64); + break; + case 32: + snd_soc_component_update_bits(component, RT5682_TDM_TCON_CTRL, + RT5682_TDM_BCLK_MS1_MASK, RT5682_TDM_BCLK_MS1_32); + break; + default: + dev_err(dai->dev, "Invalid bclk1 ratio %d\n", ratio); + return -EINVAL; + } + + return 0; +} + +static int rt5682_set_bclk2_ratio(struct snd_soc_dai *dai, unsigned int ratio) { struct snd_soc_component *component = dai->component; struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); @@ -2280,7 +2461,7 @@ static int rt5682_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) RT5682_I2S2_BCLK_MS2_32); break; default: - dev_err(dai->dev, "Invalid bclk ratio %d\n", ratio); + dev_err(dai->dev, "Invalid bclk2 ratio %d\n", ratio); return -EINVAL; } @@ -2319,12 +2500,392 @@ static int rt5682_set_bias_level(struct snd_soc_component *component, return 0; } +#ifdef CONFIG_COMMON_CLK +#define CLK_PLL2_FIN 48000000 +#define CLK_PLL2_FOUT 24576000 +#define CLK_48 48000 + +static bool rt5682_clk_check(struct rt5682_priv *rt5682) +{ + if (!rt5682->master[RT5682_AIF1]) { + dev_err(rt5682->component->dev, "sysclk/dai not set correctly\n"); + return false; + } + return true; +} + +static int rt5682_wclk_prepare(struct clk_hw *hw) +{ + struct rt5682_priv *rt5682 = + container_of(hw, struct rt5682_priv, + dai_clks_hw[RT5682_DAI_WCLK_IDX]); + struct snd_soc_component *component = rt5682->component; + struct snd_soc_dapm_context *dapm = + snd_soc_component_get_dapm(component); + + if (!rt5682_clk_check(rt5682)) + return -EINVAL; + + snd_soc_dapm_mutex_lock(dapm); + + snd_soc_dapm_force_enable_pin_unlocked(dapm, "MICBIAS"); + snd_soc_component_update_bits(component, RT5682_PWR_ANLG_1, + RT5682_PWR_MB, RT5682_PWR_MB); + snd_soc_dapm_force_enable_pin_unlocked(dapm, "I2S1"); + snd_soc_dapm_force_enable_pin_unlocked(dapm, "PLL2F"); + snd_soc_dapm_force_enable_pin_unlocked(dapm, "PLL2B"); + snd_soc_dapm_sync_unlocked(dapm); + + snd_soc_dapm_mutex_unlock(dapm); + + return 0; +} + +static void rt5682_wclk_unprepare(struct clk_hw *hw) +{ + struct rt5682_priv *rt5682 = + container_of(hw, struct rt5682_priv, + dai_clks_hw[RT5682_DAI_WCLK_IDX]); + struct snd_soc_component *component = rt5682->component; + struct snd_soc_dapm_context *dapm = + snd_soc_component_get_dapm(component); + + if (!rt5682_clk_check(rt5682)) + return; + + snd_soc_dapm_mutex_lock(dapm); + + snd_soc_dapm_disable_pin_unlocked(dapm, "MICBIAS"); + if (!rt5682->jack_type) + snd_soc_component_update_bits(component, RT5682_PWR_ANLG_1, + RT5682_PWR_MB, 0); + snd_soc_dapm_disable_pin_unlocked(dapm, "I2S1"); + snd_soc_dapm_disable_pin_unlocked(dapm, "PLL2F"); + snd_soc_dapm_disable_pin_unlocked(dapm, "PLL2B"); + snd_soc_dapm_sync_unlocked(dapm); + + snd_soc_dapm_mutex_unlock(dapm); +} + +static unsigned long rt5682_wclk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct rt5682_priv *rt5682 = + container_of(hw, struct rt5682_priv, + dai_clks_hw[RT5682_DAI_WCLK_IDX]); + + if (!rt5682_clk_check(rt5682)) + return 0; + /* + * Only accept to set wclk rate to 48kHz temporarily. + */ + return CLK_48; +} + +static long rt5682_wclk_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + struct rt5682_priv *rt5682 = + container_of(hw, struct rt5682_priv, + dai_clks_hw[RT5682_DAI_WCLK_IDX]); + + if (!rt5682_clk_check(rt5682)) + return -EINVAL; + /* + * Only accept to set wclk rate to 48kHz temporarily. + */ + return CLK_48; +} + +static int rt5682_wclk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct rt5682_priv *rt5682 = + container_of(hw, struct rt5682_priv, + dai_clks_hw[RT5682_DAI_WCLK_IDX]); + struct snd_soc_component *component = rt5682->component; + struct clk *parent_clk; + const char * const clk_name = __clk_get_name(hw->clk); + int pre_div; + + if (!rt5682_clk_check(rt5682)) + return -EINVAL; + + /* + * Whether the wclk's parent clk (mclk) exists or not, please ensure + * it is fixed or set to 48MHz before setting wclk rate. It's a + * temporary limitation. Only accept 48MHz clk as the clk provider. + * + * It will set the codec anyway by assuming mclk is 48MHz. + */ + parent_clk = clk_get_parent(hw->clk); + if (!parent_clk) + dev_warn(component->dev, + "Parent mclk of wclk not acquired in driver. Please ensure mclk was provided as %d Hz.\n", + CLK_PLL2_FIN); + + if (parent_rate != CLK_PLL2_FIN) + dev_warn(component->dev, "clk %s only support %d Hz input\n", + clk_name, CLK_PLL2_FIN); + + /* + * It's a temporary limitation. Only accept to set wclk rate to 48kHz. + * It will force wclk to 48kHz even it's not. + */ + if (rate != CLK_48) { + dev_warn(component->dev, "clk %s only support %d Hz output\n", + clk_name, CLK_48); + rate = CLK_48; + } + + /* + * To achieve the rate conversion from 48MHz to 48kHz, PLL2 is needed. + */ + rt5682_set_component_pll(component, RT5682_PLL2, RT5682_PLL2_S_MCLK, + CLK_PLL2_FIN, CLK_PLL2_FOUT); + + rt5682_set_component_sysclk(component, RT5682_SCLK_S_PLL2, 0, + CLK_PLL2_FOUT, SND_SOC_CLOCK_IN); + + pre_div = rl6231_get_clk_info(rt5682->sysclk, rate); + + snd_soc_component_update_bits(component, RT5682_ADDA_CLK_1, + RT5682_I2S_M_DIV_MASK | RT5682_I2S_CLK_SRC_MASK, + pre_div << RT5682_I2S_M_DIV_SFT | + (rt5682->sysclk_src) << RT5682_I2S_CLK_SRC_SFT); + + return 0; +} + +static unsigned long rt5682_bclk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct rt5682_priv *rt5682 = + container_of(hw, struct rt5682_priv, + dai_clks_hw[RT5682_DAI_BCLK_IDX]); + struct snd_soc_component *component = rt5682->component; + unsigned int bclks_per_wclk; + + snd_soc_component_read(component, RT5682_TDM_TCON_CTRL, + &bclks_per_wclk); + + switch (bclks_per_wclk & RT5682_TDM_BCLK_MS1_MASK) { + case RT5682_TDM_BCLK_MS1_256: + return parent_rate * 256; + case RT5682_TDM_BCLK_MS1_128: + return parent_rate * 128; + case RT5682_TDM_BCLK_MS1_64: + return parent_rate * 64; + case RT5682_TDM_BCLK_MS1_32: + return parent_rate * 32; + default: + return 0; + } +} + +static unsigned long rt5682_bclk_get_factor(unsigned long rate, + unsigned long parent_rate) +{ + unsigned long factor; + + factor = rate / parent_rate; + if (factor < 64) + return 32; + else if (factor < 128) + return 64; + else if (factor < 256) + return 128; + else + return 256; +} + +static long rt5682_bclk_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + struct rt5682_priv *rt5682 = + container_of(hw, struct rt5682_priv, + dai_clks_hw[RT5682_DAI_BCLK_IDX]); + unsigned long factor; + + if (!*parent_rate || !rt5682_clk_check(rt5682)) + return -EINVAL; + + /* + * BCLK rates are set as a multiplier of WCLK in HW. + * We don't allow changing the parent WCLK. We just do + * some rounding down based on the parent WCLK rate + * and find the appropriate multiplier of BCLK to + * get the rounded down BCLK value. + */ + factor = rt5682_bclk_get_factor(rate, *parent_rate); + + return *parent_rate * factor; +} + +static int rt5682_bclk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct rt5682_priv *rt5682 = + container_of(hw, struct rt5682_priv, + dai_clks_hw[RT5682_DAI_BCLK_IDX]); + struct snd_soc_component *component = rt5682->component; + struct snd_soc_dai *dai = NULL; + unsigned long factor; + + if (!rt5682_clk_check(rt5682)) + return -EINVAL; + + factor = rt5682_bclk_get_factor(rate, parent_rate); + + for_each_component_dais(component, dai) + if (dai->id == RT5682_AIF1) + break; + if (!dai) { + dev_err(component->dev, "dai %d not found in component\n", + RT5682_AIF1); + return -ENODEV; + } + + return rt5682_set_bclk1_ratio(dai, factor); +} + +static const struct clk_ops rt5682_dai_clk_ops[RT5682_DAI_NUM_CLKS] = { + [RT5682_DAI_WCLK_IDX] = { + .prepare = rt5682_wclk_prepare, + .unprepare = rt5682_wclk_unprepare, + .recalc_rate = rt5682_wclk_recalc_rate, + .round_rate = rt5682_wclk_round_rate, + .set_rate = rt5682_wclk_set_rate, + }, + [RT5682_DAI_BCLK_IDX] = { + .recalc_rate = rt5682_bclk_recalc_rate, + .round_rate = rt5682_bclk_round_rate, + .set_rate = rt5682_bclk_set_rate, + }, +}; + +static int rt5682_register_dai_clks(struct snd_soc_component *component) +{ + struct device *dev = component->dev; + struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); + struct rt5682_platform_data *pdata = &rt5682->pdata; + struct clk_init_data init; + struct clk *dai_clk; + struct clk_lookup *dai_clk_lookup; + struct clk_hw *dai_clk_hw; + const char *parent_name; + int i, ret; + + for (i = 0; i < RT5682_DAI_NUM_CLKS; ++i) { + dai_clk_hw = &rt5682->dai_clks_hw[i]; + + switch (i) { + case RT5682_DAI_WCLK_IDX: + /* Make MCLK the parent of WCLK */ + if (rt5682->mclk) { + parent_name = __clk_get_name(rt5682->mclk); + init.parent_names = &parent_name; + init.num_parents = 1; + } else { + init.parent_names = NULL; + init.num_parents = 0; + } + break; + case RT5682_DAI_BCLK_IDX: + /* Make WCLK the parent of BCLK */ + parent_name = __clk_get_name( + rt5682->dai_clks[RT5682_DAI_WCLK_IDX]); + init.parent_names = &parent_name; + init.num_parents = 1; + break; + default: + dev_err(dev, "Invalid clock index\n"); + ret = -EINVAL; + goto err; + } + + init.name = pdata->dai_clk_names[i]; + init.ops = &rt5682_dai_clk_ops[i]; + init.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_GATE; + dai_clk_hw->init = &init; + + dai_clk = devm_clk_register(dev, dai_clk_hw); + if (IS_ERR(dai_clk)) { + dev_warn(dev, "Failed to register %s: %ld\n", + init.name, PTR_ERR(dai_clk)); + ret = PTR_ERR(dai_clk); + goto err; + } + rt5682->dai_clks[i] = dai_clk; + + if (dev->of_node) { + devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, + dai_clk_hw); + } else { + dai_clk_lookup = clkdev_create(dai_clk, init.name, + "%s", dev_name(dev)); + if (!dai_clk_lookup) { + ret = -ENOMEM; + goto err; + } else { + rt5682->dai_clks_lookup[i] = dai_clk_lookup; + } + } + } + + return 0; + +err: + do { + if (rt5682->dai_clks_lookup[i]) + clkdev_drop(rt5682->dai_clks_lookup[i]); + } while (i-- > 0); + + return ret; +} +#endif /* CONFIG_COMMON_CLK */ + static int rt5682_probe(struct snd_soc_component *component) { struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); + struct sdw_slave *slave; + unsigned long time; +#ifdef CONFIG_COMMON_CLK + int ret; +#endif rt5682->component = component; + if (rt5682->is_sdw) { + slave = rt5682->slave; + time = wait_for_completion_timeout( + &slave->initialization_complete, + msecs_to_jiffies(RT5682_PROBE_TIMEOUT)); + if (!time) { + dev_err(&slave->dev, "Initialization not complete, timed out\n"); + return -ETIMEDOUT; + } + } else { +#ifdef CONFIG_COMMON_CLK + /* Check if MCLK provided */ + rt5682->mclk = devm_clk_get(component->dev, "mclk"); + if (IS_ERR(rt5682->mclk)) { + if (PTR_ERR(rt5682->mclk) != -ENOENT) { + ret = PTR_ERR(rt5682->mclk); + return ret; + } + rt5682->mclk = NULL; + } else { + /* Register CCF DAI clock control */ + ret = rt5682_register_dai_clks(component); + if (ret) + return ret; + } + /* Initial setup for CCF */ + rt5682->lrck[RT5682_AIF1] = CLK_48; +#endif + } + return 0; } @@ -2332,7 +2893,16 @@ static void rt5682_remove(struct snd_soc_component *component) { struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); - rt5682_reset(rt5682->regmap); +#ifdef CONFIG_COMMON_CLK + int i; + + for (i = RT5682_DAI_NUM_CLKS - 1; i >= 0; --i) { + if (rt5682->dai_clks_lookup[i]) + clkdev_drop(rt5682->dai_clks_lookup[i]); + } +#endif + + rt5682_reset(rt5682); } #ifdef CONFIG_PM @@ -2369,14 +2939,203 @@ static const struct snd_soc_dai_ops rt5682_aif1_dai_ops = { .hw_params = rt5682_hw_params, .set_fmt = rt5682_set_dai_fmt, .set_tdm_slot = rt5682_set_tdm_slot, + .set_bclk_ratio = rt5682_set_bclk1_ratio, }; static const struct snd_soc_dai_ops rt5682_aif2_dai_ops = { .hw_params = rt5682_hw_params, .set_fmt = rt5682_set_dai_fmt, - .set_bclk_ratio = rt5682_set_bclk_ratio, + .set_bclk_ratio = rt5682_set_bclk2_ratio, }; +#if IS_ENABLED(CONFIG_SND_SOC_RT5682_SDW) +struct sdw_stream_data { + struct sdw_stream_runtime *sdw_stream; +}; + +static int rt5682_set_sdw_stream(struct snd_soc_dai *dai, void *sdw_stream, + int direction) +{ + struct sdw_stream_data *stream; + + stream = kzalloc(sizeof(*stream), GFP_KERNEL); + if (!stream) + return -ENOMEM; + + stream->sdw_stream = (struct sdw_stream_runtime *)sdw_stream; + + /* Use tx_mask or rx_mask to configure stream tag and set dma_data */ + if (direction == SNDRV_PCM_STREAM_PLAYBACK) + dai->playback_dma_data = stream; + else + dai->capture_dma_data = stream; + + return 0; +} + +static void rt5682_sdw_shutdown(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct sdw_stream_data *stream; + + stream = snd_soc_dai_get_dma_data(dai, substream); + snd_soc_dai_set_dma_data(dai, substream, NULL); + kfree(stream); +} + +static int rt5682_sdw_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); + struct sdw_stream_config stream_config; + struct sdw_port_config port_config; + enum sdw_data_direction direction; + struct sdw_stream_data *stream; + int retval, port, num_channels; + unsigned int val_p = 0, val_c = 0, osr_p = 0, osr_c = 0; + + dev_dbg(dai->dev, "%s %s", __func__, dai->name); + stream = snd_soc_dai_get_dma_data(dai, substream); + + if (!stream) + return -ENOMEM; + + if (!rt5682->slave) + return -EINVAL; + + /* SoundWire specific configuration */ + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + direction = SDW_DATA_DIR_RX; + port = 1; + } else { + direction = SDW_DATA_DIR_TX; + port = 2; + } + + stream_config.frame_rate = params_rate(params); + stream_config.ch_count = params_channels(params); + stream_config.bps = snd_pcm_format_width(params_format(params)); + stream_config.direction = direction; + + num_channels = params_channels(params); + port_config.ch_mask = (1 << (num_channels)) - 1; + port_config.num = port; + + retval = sdw_stream_add_slave(rt5682->slave, &stream_config, + &port_config, 1, stream->sdw_stream); + if (retval) { + dev_err(dai->dev, "Unable to configure port\n"); + return retval; + } + + switch (params_rate(params)) { + case 48000: + val_p = RT5682_SDW_REF_1_48K; + val_c = RT5682_SDW_REF_2_48K; + break; + case 96000: + val_p = RT5682_SDW_REF_1_96K; + val_c = RT5682_SDW_REF_2_96K; + break; + case 192000: + val_p = RT5682_SDW_REF_1_192K; + val_c = RT5682_SDW_REF_2_192K; + break; + case 32000: + val_p = RT5682_SDW_REF_1_32K; + val_c = RT5682_SDW_REF_2_32K; + break; + case 24000: + val_p = RT5682_SDW_REF_1_24K; + val_c = RT5682_SDW_REF_2_24K; + break; + case 16000: + val_p = RT5682_SDW_REF_1_16K; + val_c = RT5682_SDW_REF_2_16K; + break; + case 12000: + val_p = RT5682_SDW_REF_1_12K; + val_c = RT5682_SDW_REF_2_12K; + break; + case 8000: + val_p = RT5682_SDW_REF_1_8K; + val_c = RT5682_SDW_REF_2_8K; + break; + case 44100: + val_p = RT5682_SDW_REF_1_44K; + val_c = RT5682_SDW_REF_2_44K; + break; + case 88200: + val_p = RT5682_SDW_REF_1_88K; + val_c = RT5682_SDW_REF_2_88K; + break; + case 176400: + val_p = RT5682_SDW_REF_1_176K; + val_c = RT5682_SDW_REF_2_176K; + break; + case 22050: + val_p = RT5682_SDW_REF_1_22K; + val_c = RT5682_SDW_REF_2_22K; + break; + case 11025: + val_p = RT5682_SDW_REF_1_11K; + val_c = RT5682_SDW_REF_2_11K; + break; + default: + return -EINVAL; + } + + if (params_rate(params) <= 48000) { + osr_p = RT5682_DAC_OSR_D_8; + osr_c = RT5682_ADC_OSR_D_8; + } else if (params_rate(params) <= 96000) { + osr_p = RT5682_DAC_OSR_D_4; + osr_c = RT5682_ADC_OSR_D_4; + } else { + osr_p = RT5682_DAC_OSR_D_2; + osr_c = RT5682_ADC_OSR_D_2; + } + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + regmap_update_bits(rt5682->regmap, RT5682_SDW_REF_CLK, + RT5682_SDW_REF_1_MASK, val_p); + regmap_update_bits(rt5682->regmap, RT5682_ADDA_CLK_1, + RT5682_DAC_OSR_MASK, osr_p); + } else { + regmap_update_bits(rt5682->regmap, RT5682_SDW_REF_CLK, + RT5682_SDW_REF_2_MASK, val_c); + regmap_update_bits(rt5682->regmap, RT5682_ADDA_CLK_1, + RT5682_ADC_OSR_MASK, osr_c); + } + + return retval; +} + +static int rt5682_sdw_hw_free(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct rt5682_priv *rt5682 = snd_soc_component_get_drvdata(component); + struct sdw_stream_data *stream = + snd_soc_dai_get_dma_data(dai, substream); + + if (!rt5682->slave) + return -EINVAL; + + sdw_stream_remove_slave(rt5682->slave, stream->sdw_stream); + return 0; +} + +static struct snd_soc_dai_ops rt5682_sdw_ops = { + .hw_params = rt5682_sdw_hw_params, + .hw_free = rt5682_sdw_hw_free, + .set_sdw_stream = rt5682_set_sdw_stream, + .shutdown = rt5682_sdw_shutdown, +}; +#endif + static struct snd_soc_dai_driver rt5682_dai[] = { { .name = "rt5682-aif1", @@ -2409,6 +3168,27 @@ static struct snd_soc_dai_driver rt5682_dai[] = { }, .ops = &rt5682_aif2_dai_ops, }, +#if IS_ENABLED(CONFIG_SND_SOC_RT5682_SDW) + { + .name = "rt5682-sdw", + .id = RT5682_SDW, + .playback = { + .stream_name = "SDW Playback", + .channels_min = 1, + .channels_max = 2, + .rates = RT5682_STEREO_RATES, + .formats = RT5682_FORMATS, + }, + .capture = { + .stream_name = "SDW Capture", + .channels_min = 1, + .channels_max = 2, + .rates = RT5682_STEREO_RATES, + .formats = RT5682_FORMATS, + }, + .ops = &rt5682_sdw_ops, + }, +#endif }; static const struct snd_soc_component_driver soc_component_dev_rt5682 = { @@ -2461,10 +3241,21 @@ static int rt5682_parse_dt(struct rt5682_priv *rt5682, struct device *dev) &rt5682->pdata.jd_src); device_property_read_u32(dev, "realtek,btndet-delay", &rt5682->pdata.btndet_delay); + device_property_read_u32(dev, "realtek,dmic-clk-rate-hz", + &rt5682->pdata.dmic_clk_rate); + device_property_read_u32(dev, "realtek,dmic-delay-ms", + &rt5682->pdata.dmic_delay); rt5682->pdata.ldo1_en = of_get_named_gpio(dev->of_node, "realtek,ldo1-en-gpios", 0); + if (device_property_read_string_array(dev, "clock-output-names", + rt5682->pdata.dai_clk_names, + RT5682_DAI_NUM_CLKS) < 0) + dev_warn(dev, "Using default DAI clk names: %s, %s\n", + rt5682->pdata.dai_clk_names[RT5682_DAI_WCLK_IDX], + rt5682->pdata.dai_clk_names[RT5682_DAI_BCLK_IDX]); + return 0; } @@ -2474,7 +3265,7 @@ static void rt5682_calibrate(struct rt5682_priv *rt5682) mutex_lock(&rt5682->calibrate_mutex); - rt5682_reset(rt5682->regmap); + rt5682_reset(rt5682); regmap_write(rt5682->regmap, RT5682_I2C_CTRL, 0x000f); regmap_write(rt5682->regmap, RT5682_PWR_ANLG_1, 0xa2af); usleep_range(15000, 20000); @@ -2520,6 +3311,221 @@ static void rt5682_calibrate(struct rt5682_priv *rt5682) } +#if IS_ENABLED(CONFIG_SND_SOC_RT5682_SDW) +static int rt5682_sdw_read(void *context, unsigned int reg, unsigned int *val) +{ + struct device *dev = context; + struct rt5682_priv *rt5682 = dev_get_drvdata(dev); + unsigned int data_l, data_h; + + regmap_write(rt5682->sdw_regmap, RT5682_SDW_CMD, 0); + regmap_write(rt5682->sdw_regmap, RT5682_SDW_ADDR_H, (reg >> 8) & 0xff); + regmap_write(rt5682->sdw_regmap, RT5682_SDW_ADDR_L, (reg & 0xff)); + regmap_read(rt5682->sdw_regmap, RT5682_SDW_DATA_H, &data_h); + regmap_read(rt5682->sdw_regmap, RT5682_SDW_DATA_L, &data_l); + + *val = (data_h << 8) | data_l; + + dev_vdbg(dev, "[%s] %04x => %04x\n", __func__, reg, *val); + + return 0; +} + +static int rt5682_sdw_write(void *context, unsigned int reg, unsigned int val) +{ + struct device *dev = context; + struct rt5682_priv *rt5682 = dev_get_drvdata(dev); + + regmap_write(rt5682->sdw_regmap, RT5682_SDW_CMD, 1); + regmap_write(rt5682->sdw_regmap, RT5682_SDW_ADDR_H, (reg >> 8) & 0xff); + regmap_write(rt5682->sdw_regmap, RT5682_SDW_ADDR_L, (reg & 0xff)); + regmap_write(rt5682->sdw_regmap, RT5682_SDW_DATA_H, (val >> 8) & 0xff); + regmap_write(rt5682->sdw_regmap, RT5682_SDW_DATA_L, (val & 0xff)); + + dev_vdbg(dev, "[%s] %04x <= %04x\n", __func__, reg, val); + + return 0; +} + +static const struct regmap_config rt5682_sdw_regmap = { + .reg_bits = 16, + .val_bits = 16, + .max_register = RT5682_I2C_MODE, + .volatile_reg = rt5682_volatile_register, + .readable_reg = rt5682_readable_register, + .cache_type = REGCACHE_RBTREE, + .reg_defaults = rt5682_reg, + .num_reg_defaults = ARRAY_SIZE(rt5682_reg), + .use_single_read = true, + .use_single_write = true, + .reg_read = rt5682_sdw_read, + .reg_write = rt5682_sdw_write, +}; + +int rt5682_sdw_init(struct device *dev, struct regmap *regmap, + struct sdw_slave *slave) +{ + struct rt5682_priv *rt5682; + int ret; + + rt5682 = devm_kzalloc(dev, sizeof(*rt5682), GFP_KERNEL); + if (!rt5682) + return -ENOMEM; + + dev_set_drvdata(dev, rt5682); + rt5682->slave = slave; + rt5682->sdw_regmap = regmap; + rt5682->is_sdw = true; + + rt5682->regmap = devm_regmap_init(dev, NULL, dev, &rt5682_sdw_regmap); + if (IS_ERR(rt5682->regmap)) { + ret = PTR_ERR(rt5682->regmap); + dev_err(dev, "Failed to allocate register map: %d\n", + ret); + return ret; + } + + /* + * Mark hw_init to false + * HW init will be performed when device reports present + */ + rt5682->hw_init = false; + rt5682->first_hw_init = false; + + mutex_init(&rt5682->calibrate_mutex); + INIT_DELAYED_WORK(&rt5682->jack_detect_work, + rt5682_jack_detect_handler); + + ret = devm_snd_soc_register_component(dev, &soc_component_dev_rt5682, + rt5682_dai, ARRAY_SIZE(rt5682_dai)); + + dev_dbg(&slave->dev, "%s\n", __func__); + + return ret; +} +EXPORT_SYMBOL_GPL(rt5682_sdw_init); + +int rt5682_io_init(struct device *dev, struct sdw_slave *slave) +{ + struct rt5682_priv *rt5682 = dev_get_drvdata(dev); + int ret = 0; + unsigned int val; + + if (rt5682->hw_init) + return 0; + + regmap_read(rt5682->regmap, RT5682_DEVICE_ID, &val); + if (val != DEVICE_ID) { + pr_err("Device with ID register %x is not rt5682\n", val); + return -ENODEV; + } + + /* + * PM runtime is only enabled when a Slave reports as Attached + */ + if (!rt5682->first_hw_init) { + /* set autosuspend parameters */ + pm_runtime_set_autosuspend_delay(&slave->dev, 3000); + pm_runtime_use_autosuspend(&slave->dev); + + /* update count of parent 'active' children */ + pm_runtime_set_active(&slave->dev); + + /* make sure the device does not suspend immediately */ + pm_runtime_mark_last_busy(&slave->dev); + + pm_runtime_enable(&slave->dev); + } + + pm_runtime_get_noresume(&slave->dev); + + rt5682_reset(rt5682); + + if (rt5682->first_hw_init) { + regcache_cache_only(rt5682->regmap, false); + regcache_cache_bypass(rt5682->regmap, true); + } + + rt5682_calibrate(rt5682); + + if (rt5682->first_hw_init) { + regcache_cache_bypass(rt5682->regmap, false); + regcache_mark_dirty(rt5682->regmap); + regcache_sync(rt5682->regmap); + + /* volatile registers */ + regmap_update_bits(rt5682->regmap, RT5682_CBJ_CTRL_2, + RT5682_EXT_JD_SRC, RT5682_EXT_JD_SRC_MANUAL); + + goto reinit; + } + + ret = regmap_multi_reg_write(rt5682->regmap, patch_list, + ARRAY_SIZE(patch_list)); + if (ret != 0) + dev_warn(dev, "Failed to apply regmap patch: %d\n", ret); + + regmap_write(rt5682->regmap, RT5682_DEPOP_1, 0x0000); + + regmap_update_bits(rt5682->regmap, RT5682_PWR_ANLG_1, + RT5682_LDO1_DVO_MASK | RT5682_HP_DRIVER_MASK, + RT5682_LDO1_DVO_12 | RT5682_HP_DRIVER_5X); + regmap_write(rt5682->regmap, RT5682_MICBIAS_2, 0x0380); + regmap_write(rt5682->regmap, RT5682_TEST_MODE_CTRL_1, 0x0000); + regmap_update_bits(rt5682->regmap, RT5682_BIAS_CUR_CTRL_8, + RT5682_HPA_CP_BIAS_CTRL_MASK, RT5682_HPA_CP_BIAS_3UA); + regmap_update_bits(rt5682->regmap, RT5682_CHARGE_PUMP_1, + RT5682_CP_CLK_HP_MASK, RT5682_CP_CLK_HP_300KHZ); + regmap_update_bits(rt5682->regmap, RT5682_HP_CHARGE_PUMP_1, + RT5682_PM_HP_MASK, RT5682_PM_HP_HV); + + /* Soundwire */ + regmap_write(rt5682->regmap, RT5682_PLL2_INTERNAL, 0xa266); + regmap_write(rt5682->regmap, RT5682_PLL2_CTRL_1, 0x1700); + regmap_write(rt5682->regmap, RT5682_PLL2_CTRL_2, 0x0006); + regmap_write(rt5682->regmap, RT5682_PLL2_CTRL_3, 0x2600); + regmap_write(rt5682->regmap, RT5682_PLL2_CTRL_4, 0x0c8f); + regmap_write(rt5682->regmap, RT5682_PLL_TRACK_2, 0x3000); + regmap_write(rt5682->regmap, RT5682_PLL_TRACK_3, 0x4000); + regmap_update_bits(rt5682->regmap, RT5682_GLB_CLK, + RT5682_SCLK_SRC_MASK | RT5682_PLL2_SRC_MASK, + RT5682_SCLK_SRC_PLL2 | RT5682_PLL2_SRC_SDW); + + regmap_update_bits(rt5682->regmap, RT5682_CBJ_CTRL_2, + RT5682_EXT_JD_SRC, RT5682_EXT_JD_SRC_MANUAL); + regmap_write(rt5682->regmap, RT5682_CBJ_CTRL_1, 0xd042); + regmap_update_bits(rt5682->regmap, RT5682_CBJ_CTRL_3, + RT5682_CBJ_IN_BUF_EN, RT5682_CBJ_IN_BUF_EN); + regmap_update_bits(rt5682->regmap, RT5682_SAR_IL_CMD_1, + RT5682_SAR_POW_MASK, RT5682_SAR_POW_EN); + regmap_update_bits(rt5682->regmap, RT5682_RC_CLK_CTRL, + RT5682_POW_IRQ | RT5682_POW_JDH | + RT5682_POW_ANA, RT5682_POW_IRQ | + RT5682_POW_JDH | RT5682_POW_ANA); + regmap_update_bits(rt5682->regmap, RT5682_PWR_ANLG_2, + RT5682_PWR_JDH, RT5682_PWR_JDH); + regmap_update_bits(rt5682->regmap, RT5682_IRQ_CTRL_2, + RT5682_JD1_EN_MASK | RT5682_JD1_IRQ_MASK, + RT5682_JD1_EN | RT5682_JD1_IRQ_PUL); + +reinit: + mod_delayed_work(system_power_efficient_wq, + &rt5682->jack_detect_work, msecs_to_jiffies(250)); + + /* Mark Slave initialization complete */ + rt5682->hw_init = true; + rt5682->first_hw_init = true; + + pm_runtime_mark_last_busy(&slave->dev); + pm_runtime_put_autosuspend(&slave->dev); + + dev_dbg(&slave->dev, "%s hw_init complete\n", __func__); + + return ret; +} +EXPORT_SYMBOL_GPL(rt5682_io_init); +#endif + static int rt5682_i2c_probe(struct i2c_client *i2c, const struct i2c_device_id *id) { @@ -2586,7 +3592,7 @@ static int rt5682_i2c_probe(struct i2c_client *i2c, return -ENODEV; } - rt5682_reset(rt5682->regmap); + rt5682_reset(rt5682); mutex_init(&rt5682->calibrate_mutex); rt5682_calibrate(rt5682); @@ -2651,6 +3657,8 @@ static int rt5682_i2c_probe(struct i2c_client *i2c, RT5682_CP_CLK_HP_MASK, RT5682_CP_CLK_HP_300KHZ); regmap_update_bits(rt5682->regmap, RT5682_HP_CHARGE_PUMP_1, RT5682_PM_HP_MASK, RT5682_PM_HP_HV); + regmap_update_bits(rt5682->regmap, RT5682_DMIC_CTRL_1, + RT5682_FIFO_CLK_DIV_MASK, RT5682_FIFO_CLK_DIV_2); INIT_DELAYED_WORK(&rt5682->jack_detect_work, rt5682_jack_detect_handler); @@ -2676,7 +3684,7 @@ static void rt5682_i2c_shutdown(struct i2c_client *client) { struct rt5682_priv *rt5682 = i2c_get_clientdata(client); - rt5682_reset(rt5682->regmap); + rt5682_reset(rt5682); } #ifdef CONFIG_OF diff --git a/sound/soc/codecs/rt5682.h b/sound/soc/codecs/rt5682.h index 18faaa2a49a0..0baeece84ec4 100644 --- a/sound/soc/codecs/rt5682.h +++ b/sound/soc/codecs/rt5682.h @@ -10,6 +10,12 @@ #define __RT5682_H__ #include +#include +#include +#include +#include +#include +#include #define DEVICE_ID 0x6530 @@ -177,7 +183,7 @@ #define RT5682_TEST_MODE_CTRL_4 0x0148 #define RT5682_TEST_MODE_CTRL_5 0x0149 #define RT5682_PLL1_INTERNAL 0x0150 -#define RT5682_PLL2_INTERNAL 0x0151 +#define RT5682_PLL2_INTERNAL 0x0156 #define RT5682_STO_NG2_CTRL_1 0x0160 #define RT5682_STO_NG2_CTRL_2 0x0161 #define RT5682_STO_NG2_CTRL_3 0x0162 @@ -651,6 +657,8 @@ #define RT5682_DMIC_1_EN_SFT 15 #define RT5682_DMIC_1_DIS (0x0 << 15) #define RT5682_DMIC_1_EN (0x1 << 15) +#define RT5682_FIFO_CLK_DIV_MASK (0x7 << 12) +#define RT5682_FIFO_CLK_DIV_2 (0x1 << 12) #define RT5682_DMIC_1_DP_MASK (0x3 << 4) #define RT5682_DMIC_1_DP_SFT 4 #define RT5682_DMIC_1_DP_GPIO2 (0x0 << 4) @@ -738,7 +746,7 @@ #define RT5682_ADC_OSR_D_24 (0x7 << 12) #define RT5682_ADC_OSR_D_32 (0x8 << 12) #define RT5682_ADC_OSR_D_48 (0x9 << 12) -#define RT5682_I2S_M_DIV_MASK (0xf << 12) +#define RT5682_I2S_M_DIV_MASK (0xf << 8) #define RT5682_I2S_M_DIV_SFT 8 #define RT5682_I2S_M_D_1 (0x0 << 8) #define RT5682_I2S_M_D_2 (0x1 << 8) @@ -820,6 +828,12 @@ #define RT5682_TDM_DF_PCM_B (0x3 << 11) #define RT5682_TDM_DF_PCM_A_N (0x6 << 11) #define RT5682_TDM_DF_PCM_B_N (0x7 << 11) +#define RT5682_TDM_BCLK_MS1_MASK (0x3 << 9) +#define RT5682_TDM_BCLK_MS1_SFT 9 +#define RT5682_TDM_BCLK_MS1_32 (0x0 << 9) +#define RT5682_TDM_BCLK_MS1_64 (0x1 << 9) +#define RT5682_TDM_BCLK_MS1_128 (0x2 << 9) +#define RT5682_TDM_BCLK_MS1_256 (0x3 << 9) #define RT5682_TDM_CL_MASK (0x3 << 4) #define RT5682_TDM_CL_16 (0x0 << 4) #define RT5682_TDM_CL_20 (0x1 << 4) @@ -835,8 +849,8 @@ #define RT5682_TDM_M_LP_INV (0x1 << 1) #define RT5682_TDM_MS_MASK (0x1 << 0) #define RT5682_TDM_MS_SFT 0 -#define RT5682_TDM_MS_M (0x0 << 0) -#define RT5682_TDM_MS_S (0x1 << 0) +#define RT5682_TDM_MS_S (0x0 << 0) +#define RT5682_TDM_MS_M (0x1 << 0) /* Global Clock Control (0x0080) */ #define RT5682_SCLK_SRC_MASK (0x7 << 13) @@ -1049,6 +1063,28 @@ #define RT5682_PWR_CLK1M_PD (0x0 << 8) #define RT5682_PWR_CLK1M_PU (0x1 << 8) +/* PLL2 M/N/K Code Control 1 (0x009b) */ +#define RT5682_PLL2F_K_MASK (0x1f << 8) +#define RT5682_PLL2F_K_SFT 8 +#define RT5682_PLL2B_K_MASK (0xf << 4) +#define RT5682_PLL2B_K_SFT 4 +#define RT5682_PLL2B_M_MASK (0xf << 0) + +/* PLL2 M/N/K Code Control 2 (0x009c) */ +#define RT5682_PLL2F_M_MASK (0x3f << 8) +#define RT5682_PLL2F_M_SFT 8 +#define RT5682_PLL2B_N_MASK (0x3f << 0) + +/* PLL2 M/N/K Code Control 2 (0x009d) */ +#define RT5682_PLL2F_N_MASK (0x7f << 8) +#define RT5682_PLL2F_N_SFT 8 + +/* PLL2 M/N/K Code Control 2 (0x009e) */ +#define RT5682_PLL2B_M_BP_MASK (0x1 << 11) +#define RT5682_PLL2B_M_BP_SFT 11 +#define RT5682_PLL2F_M_BP_MASK (0x1 << 7) +#define RT5682_PLL2F_M_BP_SFT 7 + /* RC Clock Control (0x009f) */ #define RT5682_POW_IRQ (0x1 << 15) #define RT5682_POW_JDH (0x1 << 14) @@ -1091,11 +1127,17 @@ #define RT5682_JD1_POL_MASK (0x1 << 13) #define RT5682_JD1_POL_NOR (0x0 << 13) #define RT5682_JD1_POL_INV (0x1 << 13) +#define RT5682_JD1_IRQ_MASK (0x1 << 10) +#define RT5682_JD1_IRQ_LEV (0x0 << 10) +#define RT5682_JD1_IRQ_PUL (0x1 << 10) /* IRQ Control 3 (0x00b8) */ #define RT5682_IL_IRQ_MASK (0x1 << 7) #define RT5682_IL_IRQ_DIS (0x0 << 7) #define RT5682_IL_IRQ_EN (0x1 << 7) +#define RT5682_IL_IRQ_TYPE_MASK (0x1 << 4) +#define RT5682_IL_IRQ_LEV (0x0 << 4) +#define RT5682_IL_IRQ_PUL (0x1 << 4) /* GPIO Control 1 (0x00c0) */ #define RT5682_GP1_PIN_MASK (0x3 << 14) @@ -1309,11 +1351,19 @@ enum { RT5682_PLL1_S_MCLK, RT5682_PLL1_S_BCLK1, RT5682_PLL1_S_RCCLK, + RT5682_PLL2_S_MCLK, +}; + +enum { + RT5682_PLL1, + RT5682_PLL2, + RT5682_PLLS, }; enum { RT5682_AIF1, RT5682_AIF2, + RT5682_SDW, RT5682_AIFS }; @@ -1329,7 +1379,49 @@ enum { RT5682_CLK_SEL_I2S2_ASRC, }; +#define RT5682_NUM_SUPPLIES 3 + +struct rt5682_priv { + struct snd_soc_component *component; + struct rt5682_platform_data pdata; + struct regmap *regmap; + struct regmap *sdw_regmap; + struct snd_soc_jack *hs_jack; + struct regulator_bulk_data supplies[RT5682_NUM_SUPPLIES]; + struct delayed_work jack_detect_work; + struct delayed_work jd_check_work; + struct mutex calibrate_mutex; + struct sdw_slave *slave; + enum sdw_slave_status status; + struct sdw_bus_params params; + bool hw_init; + bool first_hw_init; + bool is_sdw; + +#ifdef CONFIG_COMMON_CLK + struct clk_hw dai_clks_hw[RT5682_DAI_NUM_CLKS]; + struct clk_lookup *dai_clks_lookup[RT5682_DAI_NUM_CLKS]; + struct clk *dai_clks[RT5682_DAI_NUM_CLKS]; + struct clk *mclk; +#endif + + int sysclk; + int sysclk_src; + int lrck[RT5682_AIFS]; + int bclk[RT5682_AIFS]; + int master[RT5682_AIFS]; + + int pll_src[RT5682_PLLS]; + int pll_in[RT5682_PLLS]; + int pll_out[RT5682_PLLS]; + + int jack_type; +}; + int rt5682_sel_asrc_clk_src(struct snd_soc_component *component, unsigned int filter_mask, unsigned int clk_src); +int rt5682_sdw_init(struct device *dev, struct regmap *regmap, + struct sdw_slave *slave); +int rt5682_io_init(struct device *dev, struct sdw_slave *slave); #endif /* __RT5682_H__ */ diff --git a/sound/soc/codecs/tas2562.c b/sound/soc/codecs/tas2562.c index be52886a5edb..7fae88655a0f 100644 --- a/sound/soc/codecs/tas2562.c +++ b/sound/soc/codecs/tas2562.c @@ -26,6 +26,24 @@ #define TAS2562_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE |\ SNDRV_PCM_FORMAT_S32_LE) +/* DVC equation involves floating point math + * round(10^(volume in dB/20)*2^30) + * so create a lookup table for 2dB step + */ +static const unsigned int float_vol_db_lookup[] = { +0x00000d43, 0x000010b2, 0x00001505, 0x00001a67, 0x00002151, +0x000029f1, 0x000034cd, 0x00004279, 0x000053af, 0x0000695b, +0x0000695b, 0x0000a6fa, 0x0000d236, 0x000108a4, 0x00014d2a, +0x0001a36e, 0x00021008, 0x000298c0, 0x000344df, 0x00041d8f, +0x00052e5a, 0x000685c8, 0x00083621, 0x000a566d, 0x000d03a7, +0x0010624d, 0x0014a050, 0x0019f786, 0x0020b0bc, 0x0029279d, +0x0033cf8d, 0x004139d3, 0x00521d50, 0x00676044, 0x0082248a, +0x00a3d70a, 0x00ce4328, 0x0103ab3d, 0x0146e75d, 0x019b8c27, +0x02061b89, 0x028c423f, 0x03352529, 0x0409c2b0, 0x05156d68, +0x080e9f96, 0x0a24b062, 0x0cc509ab, 0x10137987, 0x143d1362, +0x197a967f, 0x2013739e, 0x28619ae9, 0x32d64617, 0x40000000 +}; + struct tas2562_data { struct snd_soc_component *component; struct gpio_desc *sdz_gpio; @@ -34,6 +52,12 @@ struct tas2562_data { struct i2c_client *client; int v_sense_slot; int i_sense_slot; + int volume_lvl; +}; + +enum tas256x_model { + TAS2562, + TAS2563, }; static int tas2562_set_bias_level(struct snd_soc_component *component, @@ -383,21 +407,81 @@ static int tas2562_dac_event(struct snd_soc_dapm_widget *w, struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(component); + int ret; switch (event) { case SND_SOC_DAPM_POST_PMU: - dev_info(tas2562->dev, "SND_SOC_DAPM_POST_PMU\n"); + ret = snd_soc_component_update_bits(component, + TAS2562_PWR_CTRL, + TAS2562_MODE_MASK, + TAS2562_MUTE); + if (ret) + goto end; break; case SND_SOC_DAPM_PRE_PMD: - dev_info(tas2562->dev, "SND_SOC_DAPM_PRE_PMD\n"); + ret = snd_soc_component_update_bits(component, + TAS2562_PWR_CTRL, + TAS2562_MODE_MASK, + TAS2562_SHUTDOWN); + if (ret) + goto end; break; default: - break; + dev_err(tas2562->dev, "Not supported evevt\n"); + return -EINVAL; } +end: + if (ret < 0) + return ret; + return 0; } +static int tas2562_volume_control_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol); + struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(component); + + ucontrol->value.integer.value[0] = tas2562->volume_lvl; + return 0; +} + +static int tas2562_volume_control_put(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol); + struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(component); + int ret; + u32 reg_val; + + reg_val = float_vol_db_lookup[ucontrol->value.integer.value[0]/2]; + ret = snd_soc_component_write(component, TAS2562_DVC_CFG4, + (reg_val & 0xff)); + if (ret) + return ret; + ret = snd_soc_component_write(component, TAS2562_DVC_CFG3, + ((reg_val >> 8) & 0xff)); + if (ret) + return ret; + ret = snd_soc_component_write(component, TAS2562_DVC_CFG2, + ((reg_val >> 16) & 0xff)); + if (ret) + return ret; + ret = snd_soc_component_write(component, TAS2562_DVC_CFG1, + ((reg_val >> 24) & 0xff)); + if (ret) + return ret; + + tas2562->volume_lvl = ucontrol->value.integer.value[0]; + + return ret; +} + +/* Digital Volume Control. From 0 dB to -110 dB in 1 dB steps */ +static const DECLARE_TLV_DB_SCALE(dvc_tlv, -11000, 100, 0); + static DECLARE_TLV_DB_SCALE(tas2562_dac_tlv, 850, 50, 0); static const struct snd_kcontrol_new isense_switch = @@ -409,14 +493,24 @@ static const struct snd_kcontrol_new vsense_switch = 1, 1); static const struct snd_kcontrol_new tas2562_snd_controls[] = { - SOC_SINGLE_TLV("Amp Gain Volume", TAS2562_PB_CFG1, 0, 0x1c, 0, + SOC_SINGLE_TLV("Amp Gain Volume", TAS2562_PB_CFG1, 1, 0x1c, 0, tas2562_dac_tlv), + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .name = "Digital Volume Control", + .index = 0, + .tlv.p = dvc_tlv, + .access = SNDRV_CTL_ELEM_ACCESS_TLV_READ | SNDRV_CTL_ELEM_ACCESS_READWRITE, + .info = snd_soc_info_volsw, + .get = tas2562_volume_control_get, + .put = tas2562_volume_control_put, + .private_value = SOC_SINGLE_VALUE(TAS2562_DVC_CFG1, 0, 110, 0, 0) , + }, }; static const struct snd_soc_dapm_widget tas2562_dapm_widgets[] = { SND_SOC_DAPM_AIF_IN("ASI1", "ASI1 Playback", 0, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_MUX("ASI1 Sel", SND_SOC_NOPM, 0, 0, &tas2562_asi1_mux), - SND_SOC_DAPM_AIF_IN("DAC IN", "Playback", 0, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_DAC_E("DAC", NULL, SND_SOC_NOPM, 0, 0, tas2562_dac_event, SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD), SND_SOC_DAPM_SWITCH("ISENSE", TAS2562_PWR_CTRL, 3, 1, &isense_switch), @@ -431,7 +525,7 @@ static const struct snd_soc_dapm_route tas2562_audio_map[] = { {"ASI1 Sel", "Left", "ASI1"}, {"ASI1 Sel", "Right", "ASI1"}, {"ASI1 Sel", "LeftRightDiv2", "ASI1"}, - { "DAC", NULL, "DAC IN" }, + { "DAC", NULL, "ASI1 Sel" }, { "OUT", NULL, "DAC" }, {"ISENSE", "Switch", "IMON"}, {"VSENSE", "Switch", "VMON"}, @@ -472,6 +566,13 @@ static struct snd_soc_dai_driver tas2562_dai[] = { .rates = SNDRV_PCM_RATE_8000_192000, .formats = TAS2562_FORMATS, }, + .capture = { + .stream_name = "ASI1 Capture", + .channels_min = 0, + .channels_max = 2, + .rates = SNDRV_PCM_RATE_8000_192000, + .formats = TAS2562_FORMATS, + }, .ops = &tas2562_speaker_dai_ops, }, }; @@ -495,6 +596,10 @@ static const struct reg_default tas2562_reg_defaults[] = { { TAS2562_PB_CFG1, 0x20 }, { TAS2562_TDM_CFG0, 0x09 }, { TAS2562_TDM_CFG1, 0x02 }, + { TAS2562_DVC_CFG1, 0x40 }, + { TAS2562_DVC_CFG2, 0x40 }, + { TAS2562_DVC_CFG3, 0x00 }, + { TAS2562_DVC_CFG4, 0x00 }, }; static const struct regmap_config tas2562_regmap_config = { @@ -564,13 +669,15 @@ static int tas2562_probe(struct i2c_client *client, } static const struct i2c_device_id tas2562_id[] = { - { "tas2562", 0 }, + { "tas2562", TAS2562 }, + { "tas2563", TAS2563 }, { } }; MODULE_DEVICE_TABLE(i2c, tas2562_id); static const struct of_device_id tas2562_of_match[] = { { .compatible = "ti,tas2562", }, + { .compatible = "ti,tas2563", }, { }, }; MODULE_DEVICE_TABLE(of, tas2562_of_match); diff --git a/sound/soc/codecs/tas2562.h b/sound/soc/codecs/tas2562.h index 62e659ab786d..28e75fc431d0 100644 --- a/sound/soc/codecs/tas2562.h +++ b/sound/soc/codecs/tas2562.h @@ -35,12 +35,14 @@ #define TAS2562_REV_ID TAS2562_REG(0, 0x7d) /* Page 2 */ -#define TAS2562_DVC_CFG1 TAS2562_REG(2, 0x01) -#define TAS2562_DVC_CFG2 TAS2562_REG(2, 0x02) +#define TAS2562_DVC_CFG1 TAS2562_REG(2, 0x0c) +#define TAS2562_DVC_CFG2 TAS2562_REG(2, 0x0d) +#define TAS2562_DVC_CFG3 TAS2562_REG(2, 0x0e) +#define TAS2562_DVC_CFG4 TAS2562_REG(2, 0x0f) #define TAS2562_RESET BIT(0) -#define TAS2562_MODE_MASK 0x3 +#define TAS2562_MODE_MASK GENMASK(1,0) #define TAS2562_ACTIVE 0x0 #define TAS2562_MUTE 0x1 #define TAS2562_SHUTDOWN 0x2 @@ -73,8 +75,8 @@ #define TAS2562_TDM_CFG2_RXWLEN_24B BIT(3) #define TAS2562_TDM_CFG2_RXWLEN_32B (BIT(2) | BIT(3)) -#define TAS2562_VSENSE_POWER_EN BIT(2) -#define TAS2562_ISENSE_POWER_EN BIT(3) +#define TAS2562_VSENSE_POWER_EN 2 +#define TAS2562_ISENSE_POWER_EN 3 #define TAS2562_TDM_CFG5_VSNS_EN BIT(6) #define TAS2562_TDM_CFG5_VSNS_SLOT_MASK GENMASK(5, 0) diff --git a/sound/soc/codecs/tlv320adcx140.c b/sound/soc/codecs/tlv320adcx140.c new file mode 100644 index 000000000000..38897568ee96 --- /dev/null +++ b/sound/soc/codecs/tlv320adcx140.c @@ -0,0 +1,920 @@ +// SPDX-License-Identifier: GPL-2.0 +// TLV320ADCX140 Sound driver +// Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tlv320adcx140.h" + +struct adcx140_priv { + struct snd_soc_component *component; + struct regulator *supply_areg; + struct gpio_desc *gpio_reset; + struct regmap *regmap; + struct device *dev; + + int micbias_vg; + + unsigned int dai_fmt; + unsigned int tdm_delay; + unsigned int slot_width; +}; + +static const struct reg_default adcx140_reg_defaults[] = { + { ADCX140_PAGE_SELECT, 0x00 }, + { ADCX140_SW_RESET, 0x00 }, + { ADCX140_SLEEP_CFG, 0x00 }, + { ADCX140_SHDN_CFG, 0x05 }, + { ADCX140_ASI_CFG0, 0x30 }, + { ADCX140_ASI_CFG1, 0x00 }, + { ADCX140_ASI_CFG2, 0x00 }, + { ADCX140_ASI_CH1, 0x00 }, + { ADCX140_ASI_CH2, 0x01 }, + { ADCX140_ASI_CH3, 0x02 }, + { ADCX140_ASI_CH4, 0x03 }, + { ADCX140_ASI_CH5, 0x04 }, + { ADCX140_ASI_CH6, 0x05 }, + { ADCX140_ASI_CH7, 0x06 }, + { ADCX140_ASI_CH8, 0x07 }, + { ADCX140_MST_CFG0, 0x02 }, + { ADCX140_MST_CFG1, 0x48 }, + { ADCX140_ASI_STS, 0xff }, + { ADCX140_CLK_SRC, 0x10 }, + { ADCX140_PDMCLK_CFG, 0x40 }, + { ADCX140_PDM_CFG, 0x00 }, + { ADCX140_GPIO_CFG0, 0x22 }, + { ADCX140_GPO_CFG1, 0x00 }, + { ADCX140_GPO_CFG2, 0x00 }, + { ADCX140_GPO_CFG3, 0x00 }, + { ADCX140_GPO_CFG4, 0x00 }, + { ADCX140_GPO_VAL, 0x00 }, + { ADCX140_GPIO_MON, 0x00 }, + { ADCX140_GPI_CFG0, 0x00 }, + { ADCX140_GPI_CFG1, 0x00 }, + { ADCX140_GPI_MON, 0x00 }, + { ADCX140_INT_CFG, 0x00 }, + { ADCX140_INT_MASK0, 0xff }, + { ADCX140_INT_LTCH0, 0x00 }, + { ADCX140_BIAS_CFG, 0x00 }, + { ADCX140_CH1_CFG0, 0x00 }, + { ADCX140_CH1_CFG1, 0x00 }, + { ADCX140_CH1_CFG2, 0xc9 }, + { ADCX140_CH1_CFG3, 0x80 }, + { ADCX140_CH1_CFG4, 0x00 }, + { ADCX140_CH2_CFG0, 0x00 }, + { ADCX140_CH2_CFG1, 0x00 }, + { ADCX140_CH2_CFG2, 0xc9 }, + { ADCX140_CH2_CFG3, 0x80 }, + { ADCX140_CH2_CFG4, 0x00 }, + { ADCX140_CH3_CFG0, 0x00 }, + { ADCX140_CH3_CFG1, 0x00 }, + { ADCX140_CH3_CFG2, 0xc9 }, + { ADCX140_CH3_CFG3, 0x80 }, + { ADCX140_CH3_CFG4, 0x00 }, + { ADCX140_CH4_CFG0, 0x00 }, + { ADCX140_CH4_CFG1, 0x00 }, + { ADCX140_CH4_CFG2, 0xc9 }, + { ADCX140_CH4_CFG3, 0x80 }, + { ADCX140_CH4_CFG4, 0x00 }, + { ADCX140_CH5_CFG2, 0xc9 }, + { ADCX140_CH5_CFG3, 0x80 }, + { ADCX140_CH5_CFG4, 0x00 }, + { ADCX140_CH6_CFG2, 0xc9 }, + { ADCX140_CH6_CFG3, 0x80 }, + { ADCX140_CH6_CFG4, 0x00 }, + { ADCX140_CH7_CFG2, 0xc9 }, + { ADCX140_CH7_CFG3, 0x80 }, + { ADCX140_CH7_CFG4, 0x00 }, + { ADCX140_CH8_CFG2, 0xc9 }, + { ADCX140_CH8_CFG3, 0x80 }, + { ADCX140_CH8_CFG4, 0x00 }, + { ADCX140_DSP_CFG0, 0x01 }, + { ADCX140_DSP_CFG1, 0x40 }, + { ADCX140_DRE_CFG0, 0x7b }, + { ADCX140_AGC_CFG0, 0xe7 }, + { ADCX140_IN_CH_EN, 0xf0 }, + { ADCX140_ASI_OUT_CH_EN, 0x00 }, + { ADCX140_PWR_CFG, 0x00 }, + { ADCX140_DEV_STS0, 0x00 }, + { ADCX140_DEV_STS1, 0x80 }, +}; + +static const struct regmap_range_cfg adcx140_ranges[] = { + { + .range_min = 0, + .range_max = 12 * 128, + .selector_reg = ADCX140_PAGE_SELECT, + .selector_mask = 0xff, + .selector_shift = 0, + .window_start = 0, + .window_len = 128, + }, +}; + +static bool adcx140_volatile(struct device *dev, unsigned int reg) +{ + switch (reg) { + case ADCX140_SW_RESET: + case ADCX140_DEV_STS0: + case ADCX140_DEV_STS1: + case ADCX140_ASI_STS: + return true; + default: + return false; + } +} + +static const struct regmap_config adcx140_i2c_regmap = { + .reg_bits = 8, + .val_bits = 8, + .reg_defaults = adcx140_reg_defaults, + .num_reg_defaults = ARRAY_SIZE(adcx140_reg_defaults), + .cache_type = REGCACHE_FLAT, + .ranges = adcx140_ranges, + .num_ranges = ARRAY_SIZE(adcx140_ranges), + .max_register = 12 * 128, + .volatile_reg = adcx140_volatile, +}; + +/* Digital Volume control. From -100 to 27 dB in 0.5 dB steps */ +static DECLARE_TLV_DB_SCALE(dig_vol_tlv, -10000, 50, 0); + +/* ADC gain. From 0 to 42 dB in 1 dB steps */ +static DECLARE_TLV_DB_SCALE(adc_tlv, 0, 100, 0); + +/* DRE Level. From -12 dB to -66 dB in 1 dB steps */ +static DECLARE_TLV_DB_SCALE(dre_thresh_tlv, -6600, 100, 0); +/* DRE Max Gain. From 2 dB to 26 dB in 2 dB steps */ +static DECLARE_TLV_DB_SCALE(dre_gain_tlv, 200, 200, 0); + +/* AGC Level. From -6 dB to -36 dB in 2 dB steps */ +static DECLARE_TLV_DB_SCALE(agc_thresh_tlv, -3600, 200, 0); +/* AGC Max Gain. From 3 dB to 42 dB in 3 dB steps */ +static DECLARE_TLV_DB_SCALE(agc_gain_tlv, 300, 300, 0); + +static const char * const decimation_filter_text[] = { + "Linear Phase", "Low Latency", "Ultra-low Latency" +}; + +static SOC_ENUM_SINGLE_DECL(decimation_filter_enum, ADCX140_DSP_CFG0, 4, + decimation_filter_text); + +static const struct snd_kcontrol_new decimation_filter_controls[] = { + SOC_DAPM_ENUM("Decimation Filter", decimation_filter_enum), +}; + +static const char * const resistor_text[] = { + "2.5 kOhm", "10 kOhm", "20 kOhm" +}; + +static SOC_ENUM_SINGLE_DECL(in1_resistor_enum, ADCX140_CH1_CFG0, 2, + resistor_text); +static SOC_ENUM_SINGLE_DECL(in2_resistor_enum, ADCX140_CH2_CFG0, 2, + resistor_text); +static SOC_ENUM_SINGLE_DECL(in3_resistor_enum, ADCX140_CH3_CFG0, 2, + resistor_text); +static SOC_ENUM_SINGLE_DECL(in4_resistor_enum, ADCX140_CH4_CFG0, 2, + resistor_text); + +static const struct snd_kcontrol_new in1_resistor_controls[] = { + SOC_DAPM_ENUM("CH1 Resistor Select", in1_resistor_enum), +}; +static const struct snd_kcontrol_new in2_resistor_controls[] = { + SOC_DAPM_ENUM("CH2 Resistor Select", in2_resistor_enum), +}; +static const struct snd_kcontrol_new in3_resistor_controls[] = { + SOC_DAPM_ENUM("CH3 Resistor Select", in3_resistor_enum), +}; +static const struct snd_kcontrol_new in4_resistor_controls[] = { + SOC_DAPM_ENUM("CH4 Resistor Select", in4_resistor_enum), +}; + +/* Analog/Digital Selection */ +static const char *adcx140_mic_sel_text[] = {"Analog", "Line In", "Digital"}; +static const char *adcx140_analog_sel_text[] = {"Analog", "Line In"}; + +static SOC_ENUM_SINGLE_DECL(adcx140_mic1p_enum, + ADCX140_CH1_CFG0, 5, + adcx140_mic_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic1p_control = +SOC_DAPM_ENUM("MIC1P MUX", adcx140_mic1p_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic1_analog_enum, + ADCX140_CH1_CFG0, 7, + adcx140_analog_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic1_analog_control = +SOC_DAPM_ENUM("MIC1 Analog MUX", adcx140_mic1_analog_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic1m_enum, + ADCX140_CH1_CFG0, 5, + adcx140_mic_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic1m_control = +SOC_DAPM_ENUM("MIC1M MUX", adcx140_mic1m_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic2p_enum, + ADCX140_CH2_CFG0, 5, + adcx140_mic_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic2p_control = +SOC_DAPM_ENUM("MIC2P MUX", adcx140_mic2p_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic2_analog_enum, + ADCX140_CH2_CFG0, 7, + adcx140_analog_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic2_analog_control = +SOC_DAPM_ENUM("MIC2 Analog MUX", adcx140_mic2_analog_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic2m_enum, + ADCX140_CH2_CFG0, 5, + adcx140_mic_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic2m_control = +SOC_DAPM_ENUM("MIC2M MUX", adcx140_mic2m_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic3p_enum, + ADCX140_CH3_CFG0, 5, + adcx140_mic_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic3p_control = +SOC_DAPM_ENUM("MIC3P MUX", adcx140_mic3p_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic3_analog_enum, + ADCX140_CH3_CFG0, 7, + adcx140_analog_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic3_analog_control = +SOC_DAPM_ENUM("MIC3 Analog MUX", adcx140_mic3_analog_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic3m_enum, + ADCX140_CH3_CFG0, 5, + adcx140_mic_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic3m_control = +SOC_DAPM_ENUM("MIC3M MUX", adcx140_mic3m_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic4p_enum, + ADCX140_CH4_CFG0, 5, + adcx140_mic_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic4p_control = +SOC_DAPM_ENUM("MIC4P MUX", adcx140_mic4p_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic4_analog_enum, + ADCX140_CH4_CFG0, 7, + adcx140_analog_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic4_analog_control = +SOC_DAPM_ENUM("MIC4 Analog MUX", adcx140_mic4_analog_enum); + +static SOC_ENUM_SINGLE_DECL(adcx140_mic4m_enum, + ADCX140_CH4_CFG0, 5, + adcx140_mic_sel_text); + +static const struct snd_kcontrol_new adcx140_dapm_mic4m_control = +SOC_DAPM_ENUM("MIC4M MUX", adcx140_mic4m_enum); + +static const struct snd_kcontrol_new adcx140_dapm_ch1_en_switch = + SOC_DAPM_SINGLE("Switch", ADCX140_ASI_OUT_CH_EN, 7, 1, 0); +static const struct snd_kcontrol_new adcx140_dapm_ch2_en_switch = + SOC_DAPM_SINGLE("Switch", ADCX140_ASI_OUT_CH_EN, 6, 1, 0); +static const struct snd_kcontrol_new adcx140_dapm_ch3_en_switch = + SOC_DAPM_SINGLE("Switch", ADCX140_ASI_OUT_CH_EN, 5, 1, 0); +static const struct snd_kcontrol_new adcx140_dapm_ch4_en_switch = + SOC_DAPM_SINGLE("Switch", ADCX140_ASI_OUT_CH_EN, 4, 1, 0); + +static const struct snd_kcontrol_new adcx140_dapm_ch1_dre_en_switch = + SOC_DAPM_SINGLE("Switch", ADCX140_CH1_CFG0, 0, 1, 0); +static const struct snd_kcontrol_new adcx140_dapm_ch2_dre_en_switch = + SOC_DAPM_SINGLE("Switch", ADCX140_CH2_CFG0, 0, 1, 0); +static const struct snd_kcontrol_new adcx140_dapm_ch3_dre_en_switch = + SOC_DAPM_SINGLE("Switch", ADCX140_CH3_CFG0, 0, 1, 0); +static const struct snd_kcontrol_new adcx140_dapm_ch4_dre_en_switch = + SOC_DAPM_SINGLE("Switch", ADCX140_CH4_CFG0, 0, 1, 0); + +static const struct snd_kcontrol_new adcx140_dapm_dre_en_switch = + SOC_DAPM_SINGLE("Switch", ADCX140_DSP_CFG1, 3, 1, 0); + +/* Output Mixer */ +static const struct snd_kcontrol_new adcx140_output_mixer_controls[] = { + SOC_DAPM_SINGLE("Digital CH1 Switch", 0, 0, 0, 0), + SOC_DAPM_SINGLE("Digital CH2 Switch", 0, 0, 0, 0), + SOC_DAPM_SINGLE("Digital CH3 Switch", 0, 0, 0, 0), + SOC_DAPM_SINGLE("Digital CH4 Switch", 0, 0, 0, 0), +}; + +static const struct snd_soc_dapm_widget adcx140_dapm_widgets[] = { + /* Analog Differential Inputs */ + SND_SOC_DAPM_INPUT("MIC1P"), + SND_SOC_DAPM_INPUT("MIC1M"), + SND_SOC_DAPM_INPUT("MIC2P"), + SND_SOC_DAPM_INPUT("MIC2M"), + SND_SOC_DAPM_INPUT("MIC3P"), + SND_SOC_DAPM_INPUT("MIC3M"), + SND_SOC_DAPM_INPUT("MIC4P"), + SND_SOC_DAPM_INPUT("MIC4M"), + + SND_SOC_DAPM_OUTPUT("CH1_OUT"), + SND_SOC_DAPM_OUTPUT("CH2_OUT"), + SND_SOC_DAPM_OUTPUT("CH3_OUT"), + SND_SOC_DAPM_OUTPUT("CH4_OUT"), + SND_SOC_DAPM_OUTPUT("CH5_OUT"), + SND_SOC_DAPM_OUTPUT("CH6_OUT"), + SND_SOC_DAPM_OUTPUT("CH7_OUT"), + SND_SOC_DAPM_OUTPUT("CH8_OUT"), + + SND_SOC_DAPM_MIXER("Output Mixer", SND_SOC_NOPM, 0, 0, + &adcx140_output_mixer_controls[0], + ARRAY_SIZE(adcx140_output_mixer_controls)), + + /* Input Selection to MIC_PGA */ + SND_SOC_DAPM_MUX("MIC1P Input Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic1p_control), + SND_SOC_DAPM_MUX("MIC2P Input Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic2p_control), + SND_SOC_DAPM_MUX("MIC3P Input Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic3p_control), + SND_SOC_DAPM_MUX("MIC4P Input Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic4p_control), + + /* Input Selection to MIC_PGA */ + SND_SOC_DAPM_MUX("MIC1 Analog Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic1_analog_control), + SND_SOC_DAPM_MUX("MIC2 Analog Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic2_analog_control), + SND_SOC_DAPM_MUX("MIC3 Analog Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic3_analog_control), + SND_SOC_DAPM_MUX("MIC4 Analog Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic4_analog_control), + + SND_SOC_DAPM_MUX("MIC1M Input Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic1m_control), + SND_SOC_DAPM_MUX("MIC2M Input Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic2m_control), + SND_SOC_DAPM_MUX("MIC3M Input Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic3m_control), + SND_SOC_DAPM_MUX("MIC4M Input Mux", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_mic4m_control), + + SND_SOC_DAPM_PGA("MIC_GAIN_CTL_CH1", SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_PGA("MIC_GAIN_CTL_CH2", SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_PGA("MIC_GAIN_CTL_CH3", SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_PGA("MIC_GAIN_CTL_CH4", SND_SOC_NOPM, 0, 0, NULL, 0), + + SND_SOC_DAPM_ADC("CH1_ADC", "CH1 Capture", ADCX140_IN_CH_EN, 7, 0), + SND_SOC_DAPM_ADC("CH2_ADC", "CH2 Capture", ADCX140_IN_CH_EN, 6, 0), + SND_SOC_DAPM_ADC("CH3_ADC", "CH3 Capture", ADCX140_IN_CH_EN, 5, 0), + SND_SOC_DAPM_ADC("CH4_ADC", "CH4 Capture", ADCX140_IN_CH_EN, 4, 0), + + SND_SOC_DAPM_SWITCH("CH1_ASI_EN", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_ch1_en_switch), + SND_SOC_DAPM_SWITCH("CH2_ASI_EN", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_ch2_en_switch), + SND_SOC_DAPM_SWITCH("CH3_ASI_EN", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_ch3_en_switch), + SND_SOC_DAPM_SWITCH("CH4_ASI_EN", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_ch4_en_switch), + + SND_SOC_DAPM_SWITCH("DRE_ENABLE", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_dre_en_switch), + + SND_SOC_DAPM_SWITCH("CH1_DRE_EN", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_ch1_dre_en_switch), + SND_SOC_DAPM_SWITCH("CH2_DRE_EN", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_ch2_dre_en_switch), + SND_SOC_DAPM_SWITCH("CH3_DRE_EN", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_ch3_dre_en_switch), + SND_SOC_DAPM_SWITCH("CH4_DRE_EN", SND_SOC_NOPM, 0, 0, + &adcx140_dapm_ch4_dre_en_switch), + + SND_SOC_DAPM_MUX("IN1 Analog Mic Resistor", SND_SOC_NOPM, 0, 0, + in1_resistor_controls), + SND_SOC_DAPM_MUX("IN2 Analog Mic Resistor", SND_SOC_NOPM, 0, 0, + in2_resistor_controls), + SND_SOC_DAPM_MUX("IN3 Analog Mic Resistor", SND_SOC_NOPM, 0, 0, + in3_resistor_controls), + SND_SOC_DAPM_MUX("IN4 Analog Mic Resistor", SND_SOC_NOPM, 0, 0, + in4_resistor_controls), + + SND_SOC_DAPM_MUX("Decimation Filter", SND_SOC_NOPM, 0, 0, + decimation_filter_controls), +}; + +static const struct snd_soc_dapm_route adcx140_audio_map[] = { + /* Outputs */ + {"CH1_OUT", NULL, "Output Mixer"}, + {"CH2_OUT", NULL, "Output Mixer"}, + {"CH3_OUT", NULL, "Output Mixer"}, + {"CH4_OUT", NULL, "Output Mixer"}, + + {"CH1_ASI_EN", "Switch", "CH1_ADC"}, + {"CH2_ASI_EN", "Switch", "CH2_ADC"}, + {"CH3_ASI_EN", "Switch", "CH3_ADC"}, + {"CH4_ASI_EN", "Switch", "CH4_ADC"}, + + {"Decimation Filter", "Linear Phase", "DRE_ENABLE"}, + {"Decimation Filter", "Low Latency", "DRE_ENABLE"}, + {"Decimation Filter", "Ultra-low Latency", "DRE_ENABLE"}, + + {"DRE_ENABLE", "Switch", "CH1_DRE_EN"}, + {"DRE_ENABLE", "Switch", "CH2_DRE_EN"}, + {"DRE_ENABLE", "Switch", "CH3_DRE_EN"}, + {"DRE_ENABLE", "Switch", "CH4_DRE_EN"}, + + {"CH1_DRE_EN", "Switch", "CH1_ADC"}, + {"CH2_DRE_EN", "Switch", "CH2_ADC"}, + {"CH3_DRE_EN", "Switch", "CH3_ADC"}, + {"CH4_DRE_EN", "Switch", "CH4_ADC"}, + + /* Mic input */ + {"CH1_ADC", NULL, "MIC_GAIN_CTL_CH1"}, + {"CH2_ADC", NULL, "MIC_GAIN_CTL_CH2"}, + {"CH3_ADC", NULL, "MIC_GAIN_CTL_CH3"}, + {"CH4_ADC", NULL, "MIC_GAIN_CTL_CH4"}, + + {"MIC_GAIN_CTL_CH1", NULL, "IN1 Analog Mic Resistor"}, + {"MIC_GAIN_CTL_CH1", NULL, "IN1 Analog Mic Resistor"}, + {"MIC_GAIN_CTL_CH2", NULL, "IN2 Analog Mic Resistor"}, + {"MIC_GAIN_CTL_CH2", NULL, "IN2 Analog Mic Resistor"}, + {"MIC_GAIN_CTL_CH3", NULL, "IN3 Analog Mic Resistor"}, + {"MIC_GAIN_CTL_CH3", NULL, "IN3 Analog Mic Resistor"}, + {"MIC_GAIN_CTL_CH4", NULL, "IN4 Analog Mic Resistor"}, + {"MIC_GAIN_CTL_CH4", NULL, "IN4 Analog Mic Resistor"}, + + {"IN1 Analog Mic Resistor", "2.5 kOhm", "MIC1P Input Mux"}, + {"IN1 Analog Mic Resistor", "10 kOhm", "MIC1P Input Mux"}, + {"IN1 Analog Mic Resistor", "20 kOhm", "MIC1P Input Mux"}, + + {"IN1 Analog Mic Resistor", "2.5 kOhm", "MIC1M Input Mux"}, + {"IN1 Analog Mic Resistor", "10 kOhm", "MIC1M Input Mux"}, + {"IN1 Analog Mic Resistor", "20 kOhm", "MIC1M Input Mux"}, + + {"IN2 Analog Mic Resistor", "2.5 kOhm", "MIC2P Input Mux"}, + {"IN2 Analog Mic Resistor", "10 kOhm", "MIC2P Input Mux"}, + {"IN2 Analog Mic Resistor", "20 kOhm", "MIC2P Input Mux"}, + + {"IN2 Analog Mic Resistor", "2.5 kOhm", "MIC2M Input Mux"}, + {"IN2 Analog Mic Resistor", "10 kOhm", "MIC2M Input Mux"}, + {"IN2 Analog Mic Resistor", "20 kOhm", "MIC2M Input Mux"}, + + {"IN3 Analog Mic Resistor", "2.5 kOhm", "MIC3P Input Mux"}, + {"IN3 Analog Mic Resistor", "10 kOhm", "MIC3P Input Mux"}, + {"IN3 Analog Mic Resistor", "20 kOhm", "MIC3P Input Mux"}, + + {"IN3 Analog Mic Resistor", "2.5 kOhm", "MIC3M Input Mux"}, + {"IN3 Analog Mic Resistor", "10 kOhm", "MIC3M Input Mux"}, + {"IN3 Analog Mic Resistor", "20 kOhm", "MIC3M Input Mux"}, + + {"IN4 Analog Mic Resistor", "2.5 kOhm", "MIC4P Input Mux"}, + {"IN4 Analog Mic Resistor", "10 kOhm", "MIC4P Input Mux"}, + {"IN4 Analog Mic Resistor", "20 kOhm", "MIC4P Input Mux"}, + + {"IN4 Analog Mic Resistor", "2.5 kOhm", "MIC4M Input Mux"}, + {"IN4 Analog Mic Resistor", "10 kOhm", "MIC4M Input Mux"}, + {"IN4 Analog Mic Resistor", "20 kOhm", "MIC4M Input Mux"}, + + {"MIC1 Analog Mux", "Line In", "MIC1P"}, + {"MIC2 Analog Mux", "Line In", "MIC2P"}, + {"MIC3 Analog Mux", "Line In", "MIC3P"}, + {"MIC4 Analog Mux", "Line In", "MIC4P"}, + + {"MIC1P Input Mux", "Analog", "MIC1P"}, + {"MIC1M Input Mux", "Analog", "MIC1M"}, + {"MIC2P Input Mux", "Analog", "MIC2P"}, + {"MIC2M Input Mux", "Analog", "MIC2M"}, + {"MIC3P Input Mux", "Analog", "MIC3P"}, + {"MIC3M Input Mux", "Analog", "MIC3M"}, + {"MIC4P Input Mux", "Analog", "MIC4P"}, + {"MIC4M Input Mux", "Analog", "MIC4M"}, +}; + +static const struct snd_kcontrol_new adcx140_snd_controls[] = { + SOC_SINGLE_TLV("Analog CH1 Mic Gain Volume", ADCX140_CH1_CFG1, 2, 42, 0, + adc_tlv), + SOC_SINGLE_TLV("Analog CH2 Mic Gain Volume", ADCX140_CH1_CFG2, 2, 42, 0, + adc_tlv), + SOC_SINGLE_TLV("Analog CH3 Mic Gain Volume", ADCX140_CH1_CFG3, 2, 42, 0, + adc_tlv), + SOC_SINGLE_TLV("Analog CH4 Mic Gain Volume", ADCX140_CH1_CFG4, 2, 42, 0, + adc_tlv), + + SOC_SINGLE_TLV("DRE Threshold", ADCX140_DRE_CFG0, 4, 9, 0, + dre_thresh_tlv), + SOC_SINGLE_TLV("DRE Max Gain", ADCX140_DRE_CFG0, 0, 12, 0, + dre_gain_tlv), + + SOC_SINGLE_TLV("AGC Threshold", ADCX140_AGC_CFG0, 4, 15, 0, + agc_thresh_tlv), + SOC_SINGLE_TLV("AGC Max Gain", ADCX140_AGC_CFG0, 0, 13, 0, + agc_gain_tlv), + + SOC_SINGLE_TLV("Digital CH1 Out Volume", ADCX140_CH1_CFG2, + 0, 0xff, 0, dig_vol_tlv), + SOC_SINGLE_TLV("Digital CH2 Out Volume", ADCX140_CH2_CFG2, + 0, 0xff, 0, dig_vol_tlv), + SOC_SINGLE_TLV("Digital CH3 Out Volume", ADCX140_CH3_CFG2, + 0, 0xff, 0, dig_vol_tlv), + SOC_SINGLE_TLV("Digital CH4 Out Volume", ADCX140_CH4_CFG2, + 0, 0xff, 0, dig_vol_tlv), + SOC_SINGLE_TLV("Digital CH5 Out Volume", ADCX140_CH5_CFG2, + 0, 0xff, 0, dig_vol_tlv), + SOC_SINGLE_TLV("Digital CH6 Out Volume", ADCX140_CH6_CFG2, + 0, 0xff, 0, dig_vol_tlv), + SOC_SINGLE_TLV("Digital CH7 Out Volume", ADCX140_CH7_CFG2, + 0, 0xff, 0, dig_vol_tlv), + SOC_SINGLE_TLV("Digital CH8 Out Volume", ADCX140_CH8_CFG2, + 0, 0xff, 0, dig_vol_tlv), +}; + +static int adcx140_reset(struct adcx140_priv *adcx140) +{ + int ret = 0; + + if (adcx140->gpio_reset) { + gpiod_direction_output(adcx140->gpio_reset, 0); + /* 8.4.1: wait for hw shutdown (25ms) + >= 1ms */ + usleep_range(30000, 100000); + gpiod_direction_output(adcx140->gpio_reset, 1); + } else { + ret = regmap_write(adcx140->regmap, ADCX140_SW_RESET, + ADCX140_RESET); + } + + /* 8.4.2: wait >= 10 ms after entering sleep mode. */ + usleep_range(10000, 100000); + + return 0; +} + +static int adcx140_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + u8 data = 0; + + switch (params_width(params)) { + case 16: + data = ADCX140_16_BIT_WORD; + break; + case 20: + data = ADCX140_20_BIT_WORD; + break; + case 24: + data = ADCX140_24_BIT_WORD; + break; + case 32: + data = ADCX140_32_BIT_WORD; + break; + default: + dev_err(component->dev, "%s: Unsupported width %d\n", + __func__, params_width(params)); + return -EINVAL; + } + + snd_soc_component_update_bits(component, ADCX140_ASI_CFG0, + ADCX140_WORD_LEN_MSK, data); + + return 0; +} + +static int adcx140_set_dai_fmt(struct snd_soc_dai *codec_dai, + unsigned int fmt) +{ + struct snd_soc_component *component = codec_dai->component; + struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(component); + u8 iface_reg1 = 0; + u8 iface_reg2 = 0; + + /* set master/slave audio interface */ + switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { + case SND_SOC_DAIFMT_CBM_CFM: + iface_reg2 |= ADCX140_BCLK_FSYNC_MASTER; + break; + case SND_SOC_DAIFMT_CBS_CFS: + break; + case SND_SOC_DAIFMT_CBS_CFM: + case SND_SOC_DAIFMT_CBM_CFS: + default: + dev_err(component->dev, "Invalid DAI master/slave interface\n"); + return -EINVAL; + } + + /* signal polarity */ + switch (fmt & SND_SOC_DAIFMT_INV_MASK) { + case SND_SOC_DAIFMT_NB_IF: + iface_reg1 |= ADCX140_FSYNCINV_BIT; + break; + case SND_SOC_DAIFMT_IB_IF: + iface_reg1 |= ADCX140_BCLKINV_BIT | ADCX140_FSYNCINV_BIT; + break; + case SND_SOC_DAIFMT_IB_NF: + iface_reg1 |= ADCX140_BCLKINV_BIT; + break; + case SND_SOC_DAIFMT_NB_NF: + break; + default: + dev_err(component->dev, "Invalid DAI clock signal polarity\n"); + return -EINVAL; + } + + /* interface format */ + switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { + case SND_SOC_DAIFMT_I2S: + iface_reg1 |= ADCX140_I2S_MODE_BIT; + break; + case SND_SOC_DAIFMT_LEFT_J: + iface_reg1 |= ADCX140_LEFT_JUST_BIT; + break; + case SND_SOC_DAIFMT_DSP_A: + case SND_SOC_DAIFMT_DSP_B: + break; + default: + dev_err(component->dev, "Invalid DAI interface format\n"); + return -EINVAL; + } + + adcx140->dai_fmt = fmt & SND_SOC_DAIFMT_FORMAT_MASK; + + snd_soc_component_update_bits(component, ADCX140_ASI_CFG0, + ADCX140_FSYNCINV_BIT | + ADCX140_BCLKINV_BIT | + ADCX140_ASI_FORMAT_MSK, + iface_reg1); + snd_soc_component_update_bits(component, ADCX140_MST_CFG0, + ADCX140_BCLK_FSYNC_MASTER, iface_reg2); + + return 0; +} + +static int adcx140_set_dai_tdm_slot(struct snd_soc_dai *codec_dai, + unsigned int tx_mask, unsigned int rx_mask, + int slots, int slot_width) +{ + struct snd_soc_component *component = codec_dai->component; + struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(component); + unsigned int lsb; + + if (tx_mask != rx_mask) { + dev_err(component->dev, "tx and rx masks must be symmetric\n"); + return -EINVAL; + } + + /* TDM based on DSP mode requires slots to be adjacent */ + lsb = __ffs(tx_mask); + if ((lsb + 1) != __fls(tx_mask)) { + dev_err(component->dev, "Invalid mask, slots must be adjacent\n"); + return -EINVAL; + } + + switch (slot_width) { + case 16: + case 20: + case 24: + case 32: + break; + default: + dev_err(component->dev, "Unsupported slot width %d\n", slot_width); + return -EINVAL; + } + + adcx140->tdm_delay = lsb; + adcx140->slot_width = slot_width; + + return 0; +} + +static int adcx140_prepare(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(component); + int offset = 0; + int width = adcx140->slot_width; + + if (!width) + width = substream->runtime->sample_bits; + + /* TDM slot selection only valid in DSP_A/_B mode */ + if (adcx140->dai_fmt == SND_SOC_DAIFMT_DSP_A) + offset += (adcx140->tdm_delay * width + 1); + else if (adcx140->dai_fmt == SND_SOC_DAIFMT_DSP_B) + offset += adcx140->tdm_delay * width; + + /* Configure data offset */ + snd_soc_component_update_bits(component, ADCX140_ASI_CFG1, + ADCX140_TX_OFFSET_MASK, offset); + + return 0; +} + +static const struct snd_soc_dai_ops adcx140_dai_ops = { + .hw_params = adcx140_hw_params, + .set_fmt = adcx140_set_dai_fmt, + .prepare = adcx140_prepare, + .set_tdm_slot = adcx140_set_dai_tdm_slot, +}; + +static int adcx140_codec_probe(struct snd_soc_component *component) +{ + struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(component); + int sleep_cfg_val = ADCX140_WAKE_DEV; + u8 bias_source; + u8 vref_source; + int ret; + + ret = device_property_read_u8(adcx140->dev, "ti,mic-bias-source", + &bias_source); + if (ret) + bias_source = ADCX140_MIC_BIAS_VAL_VREF; + + if (bias_source < ADCX140_MIC_BIAS_VAL_VREF || + bias_source > ADCX140_MIC_BIAS_VAL_AVDD) { + dev_err(adcx140->dev, "Mic Bias source value is invalid\n"); + return -EINVAL; + } + + ret = device_property_read_u8(adcx140->dev, "ti,vref-source", + &vref_source); + if (ret) + vref_source = ADCX140_MIC_BIAS_VREF_275V; + + if (vref_source < ADCX140_MIC_BIAS_VREF_275V || + vref_source > ADCX140_MIC_BIAS_VREF_1375V) { + dev_err(adcx140->dev, "Mic Bias source value is invalid\n"); + return -EINVAL; + } + + bias_source |= vref_source; + + ret = adcx140_reset(adcx140); + if (ret) + goto out; + + if(adcx140->supply_areg == NULL) + sleep_cfg_val |= ADCX140_AREG_INTERNAL; + + ret = regmap_write(adcx140->regmap, ADCX140_SLEEP_CFG, sleep_cfg_val); + if (ret) { + dev_err(adcx140->dev, "setting sleep config failed %d\n", ret); + goto out; + } + + /* 8.4.3: Wait >= 1ms after entering active mode. */ + usleep_range(1000, 100000); + + ret = regmap_update_bits(adcx140->regmap, ADCX140_BIAS_CFG, + ADCX140_MIC_BIAS_VAL_MSK | + ADCX140_MIC_BIAS_VREF_MSK, bias_source); + if (ret) + dev_err(adcx140->dev, "setting MIC bias failed %d\n", ret); +out: + return ret; +} + +static int adcx140_set_bias_level(struct snd_soc_component *component, + enum snd_soc_bias_level level) +{ + struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(component); + int pwr_cfg = 0; + + switch (level) { + case SND_SOC_BIAS_ON: + case SND_SOC_BIAS_PREPARE: + case SND_SOC_BIAS_STANDBY: + pwr_cfg = ADCX140_PWR_CFG_BIAS_PDZ | ADCX140_PWR_CFG_PLL_PDZ | + ADCX140_PWR_CFG_ADC_PDZ; + break; + case SND_SOC_BIAS_OFF: + pwr_cfg = 0x0; + break; + } + + return regmap_write(adcx140->regmap, ADCX140_PWR_CFG, pwr_cfg); +} + +static const struct snd_soc_component_driver soc_codec_driver_adcx140 = { + .probe = adcx140_codec_probe, + .set_bias_level = adcx140_set_bias_level, + .controls = adcx140_snd_controls, + .num_controls = ARRAY_SIZE(adcx140_snd_controls), + .dapm_widgets = adcx140_dapm_widgets, + .num_dapm_widgets = ARRAY_SIZE(adcx140_dapm_widgets), + .dapm_routes = adcx140_audio_map, + .num_dapm_routes = ARRAY_SIZE(adcx140_audio_map), + .suspend_bias_off = 1, + .idle_bias_on = 0, + .use_pmdown_time = 1, + .endianness = 1, + .non_legacy_dai_naming = 1, +}; + +static struct snd_soc_dai_driver adcx140_dai_driver[] = { + { + .name = "tlv320adcx140-codec", + .capture = { + .stream_name = "Capture", + .channels_min = 2, + .channels_max = ADCX140_MAX_CHANNELS, + .rates = ADCX140_RATES, + .formats = ADCX140_FORMATS, + }, + .ops = &adcx140_dai_ops, + .symmetric_rates = 1, + } +}; + +static const struct of_device_id tlv320adcx140_of_match[] = { + { .compatible = "ti,tlv320adc3140" }, + { .compatible = "ti,tlv320adc5140" }, + { .compatible = "ti,tlv320adc6140" }, + {}, +}; +MODULE_DEVICE_TABLE(of, tlv320adcx140_of_match); + +static int adcx140_i2c_probe(struct i2c_client *i2c, + const struct i2c_device_id *id) +{ + struct adcx140_priv *adcx140; + int ret; + + adcx140 = devm_kzalloc(&i2c->dev, sizeof(*adcx140), GFP_KERNEL); + if (!adcx140) + return -ENOMEM; + + adcx140->gpio_reset = devm_gpiod_get_optional(adcx140->dev, + "reset", GPIOD_OUT_LOW); + if (IS_ERR(adcx140->gpio_reset)) + dev_info(&i2c->dev, "Reset GPIO not defined\n"); + + adcx140->supply_areg = devm_regulator_get_optional(adcx140->dev, + "areg"); + if (IS_ERR(adcx140->supply_areg)) { + if (PTR_ERR(adcx140->supply_areg) == -EPROBE_DEFER) + return -EPROBE_DEFER; + else + adcx140->supply_areg = NULL; + } else { + ret = regulator_enable(adcx140->supply_areg); + if (ret) { + dev_err(adcx140->dev, "Failed to enable areg\n"); + return ret; + } + } + + adcx140->regmap = devm_regmap_init_i2c(i2c, &adcx140_i2c_regmap); + if (IS_ERR(adcx140->regmap)) { + ret = PTR_ERR(adcx140->regmap); + dev_err(&i2c->dev, "Failed to allocate register map: %d\n", + ret); + return ret; + } + adcx140->dev = &i2c->dev; + i2c_set_clientdata(i2c, adcx140); + + return devm_snd_soc_register_component(&i2c->dev, + &soc_codec_driver_adcx140, + adcx140_dai_driver, 1); +} + +static const struct i2c_device_id adcx140_i2c_id[] = { + { "tlv320adc3140", 0 }, + { "tlv320adc5140", 1 }, + { "tlv320adc6140", 2 }, + {} +}; +MODULE_DEVICE_TABLE(i2c, adcx140_i2c_id); + +static struct i2c_driver adcx140_i2c_driver = { + .driver = { + .name = "tlv320adcx140-codec", + .of_match_table = of_match_ptr(tlv320adcx140_of_match), + }, + .probe = adcx140_i2c_probe, + .id_table = adcx140_i2c_id, +}; +module_i2c_driver(adcx140_i2c_driver); + +MODULE_AUTHOR("Dan Murphy "); +MODULE_DESCRIPTION("ASoC TLV320ADCX140 CODEC Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/codecs/tlv320adcx140.h b/sound/soc/codecs/tlv320adcx140.h new file mode 100644 index 000000000000..6d055e55909e --- /dev/null +++ b/sound/soc/codecs/tlv320adcx140.h @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0 +// TLV320ADCX104 Sound driver +// Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/ + +#ifndef _TLV320ADCX140_H +#define _TLV320ADCX140_H + +#define ADCX140_RATES (SNDRV_PCM_RATE_44100 | \ + SNDRV_PCM_RATE_48000) + +#define ADCX140_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | \ + SNDRV_PCM_FMTBIT_S20_3LE | \ + SNDRV_PCM_FMTBIT_S24_3LE | \ + SNDRV_PCM_FMTBIT_S24_LE | \ + SNDRV_PCM_FMTBIT_S32_LE) + +#define ADCX140_PAGE_SELECT 0x00 +#define ADCX140_SW_RESET 0x01 +#define ADCX140_SLEEP_CFG 0x02 +#define ADCX140_SHDN_CFG 0x05 +#define ADCX140_ASI_CFG0 0x07 +#define ADCX140_ASI_CFG1 0x08 +#define ADCX140_ASI_CFG2 0x09 +#define ADCX140_ASI_CH1 0x0b +#define ADCX140_ASI_CH2 0x0c +#define ADCX140_ASI_CH3 0x0d +#define ADCX140_ASI_CH4 0x0e +#define ADCX140_ASI_CH5 0x0f +#define ADCX140_ASI_CH6 0x10 +#define ADCX140_ASI_CH7 0x11 +#define ADCX140_ASI_CH8 0x12 +#define ADCX140_MST_CFG0 0x13 +#define ADCX140_MST_CFG1 0x14 +#define ADCX140_ASI_STS 0x15 +#define ADCX140_CLK_SRC 0x16 +#define ADCX140_PDMCLK_CFG 0x1f +#define ADCX140_PDM_CFG 0x20 +#define ADCX140_GPIO_CFG0 0x21 +#define ADCX140_GPO_CFG1 0x22 +#define ADCX140_GPO_CFG2 0x23 +#define ADCX140_GPO_CFG3 0x24 +#define ADCX140_GPO_CFG4 0x25 +#define ADCX140_GPO_VAL 0x29 +#define ADCX140_GPIO_MON 0x2a +#define ADCX140_GPI_CFG0 0x2b +#define ADCX140_GPI_CFG1 0x2c +#define ADCX140_GPI_MON 0x2f +#define ADCX140_INT_CFG 0x32 +#define ADCX140_INT_MASK0 0x33 +#define ADCX140_INT_LTCH0 0x36 +#define ADCX140_BIAS_CFG 0x3b +#define ADCX140_CH1_CFG0 0x3c +#define ADCX140_CH1_CFG1 0x3d +#define ADCX140_CH1_CFG2 0x3e +#define ADCX140_CH1_CFG3 0x3f +#define ADCX140_CH1_CFG4 0x40 +#define ADCX140_CH2_CFG0 0x41 +#define ADCX140_CH2_CFG1 0x42 +#define ADCX140_CH2_CFG2 0x43 +#define ADCX140_CH2_CFG3 0x44 +#define ADCX140_CH2_CFG4 0x45 +#define ADCX140_CH3_CFG0 0x46 +#define ADCX140_CH3_CFG1 0x47 +#define ADCX140_CH3_CFG2 0x48 +#define ADCX140_CH3_CFG3 0x49 +#define ADCX140_CH3_CFG4 0x4a +#define ADCX140_CH4_CFG0 0x4b +#define ADCX140_CH4_CFG1 0x4c +#define ADCX140_CH4_CFG2 0x4d +#define ADCX140_CH4_CFG3 0x4e +#define ADCX140_CH4_CFG4 0x4f +#define ADCX140_CH5_CFG2 0x52 +#define ADCX140_CH5_CFG3 0x53 +#define ADCX140_CH5_CFG4 0x54 +#define ADCX140_CH6_CFG2 0x57 +#define ADCX140_CH6_CFG3 0x58 +#define ADCX140_CH6_CFG4 0x59 +#define ADCX140_CH7_CFG2 0x5c +#define ADCX140_CH7_CFG3 0x5d +#define ADCX140_CH7_CFG4 0x5e +#define ADCX140_CH8_CFG2 0x61 +#define ADCX140_CH8_CFG3 0x62 +#define ADCX140_CH8_CFG4 0x63 +#define ADCX140_DSP_CFG0 0x6b +#define ADCX140_DSP_CFG1 0x6c +#define ADCX140_DRE_CFG0 0x6d +#define ADCX140_AGC_CFG0 0x70 +#define ADCX140_IN_CH_EN 0x73 +#define ADCX140_ASI_OUT_CH_EN 0x74 +#define ADCX140_PWR_CFG 0x75 +#define ADCX140_DEV_STS0 0x76 +#define ADCX140_DEV_STS1 0x77 + +#define ADCX140_RESET BIT(0) + +#define ADCX140_WAKE_DEV BIT(0) +#define ADCX140_AREG_INTERNAL BIT(7) + +#define ADCX140_BCLKINV_BIT BIT(2) +#define ADCX140_FSYNCINV_BIT BIT(3) +#define ADCX140_INV_MSK (ADCX140_BCLKINV_BIT | ADCX140_FSYNCINV_BIT) +#define ADCX140_BCLK_FSYNC_MASTER BIT(7) +#define ADCX140_I2S_MODE_BIT BIT(6) +#define ADCX140_LEFT_JUST_BIT BIT(7) +#define ADCX140_ASI_FORMAT_MSK (ADCX140_I2S_MODE_BIT | ADCX140_LEFT_JUST_BIT) + +#define ADCX140_16_BIT_WORD 0x0 +#define ADCX140_20_BIT_WORD BIT(4) +#define ADCX140_24_BIT_WORD BIT(5) +#define ADCX140_32_BIT_WORD (BIT(4) | BIT(5)) +#define ADCX140_WORD_LEN_MSK 0x30 + +#define ADCX140_MAX_CHANNELS 8 + +#define ADCX140_MIC_BIAS_VAL_VREF 0 +#define ADCX140_MIC_BIAS_VAL_VREF_1096 1 +#define ADCX140_MIC_BIAS_VAL_AVDD 6 +#define ADCX140_MIC_BIAS_VAL_MSK GENMASK(6, 4) + +#define ADCX140_MIC_BIAS_VREF_275V 0 +#define ADCX140_MIC_BIAS_VREF_25V 1 +#define ADCX140_MIC_BIAS_VREF_1375V 2 +#define ADCX140_MIC_BIAS_VREF_MSK GENMASK(1, 0) + +#define ADCX140_PWR_CFG_BIAS_PDZ BIT(7) +#define ADCX140_PWR_CFG_ADC_PDZ BIT(6) +#define ADCX140_PWR_CFG_PLL_PDZ BIT(5) + +#define ADCX140_TX_OFFSET_MASK GENMASK(4, 0) + +#endif /* _TLV320ADCX140_ */ diff --git a/sound/soc/codecs/wcd9335.c b/sound/soc/codecs/wcd9335.c index f11ffa28683b..700cc1212770 100644 --- a/sound/soc/codecs/wcd9335.c +++ b/sound/soc/codecs/wcd9335.c @@ -4926,11 +4926,11 @@ static const struct regmap_range_cfg wcd9335_ranges[] = { .name = "WCD9335", .range_min = 0x0, .range_max = WCD9335_MAX_REGISTER, - .selector_reg = WCD9335_REG(0x0, 0), + .selector_reg = WCD9335_SEL_REGISTER, .selector_mask = 0xff, .selector_shift = 0, - .window_start = 0x0, - .window_len = 0x1000, + .window_start = 0x800, + .window_len = 0x100, }, }; @@ -4968,12 +4968,12 @@ static const struct regmap_range_cfg wcd9335_ifc_ranges[] = { { .name = "WCD9335-IFC-DEV", .range_min = 0x0, - .range_max = WCD9335_REG(0, 0x7ff), - .selector_reg = WCD9335_REG(0, 0x0), - .selector_mask = 0xff, + .range_max = WCD9335_MAX_REGISTER, + .selector_reg = WCD9335_SEL_REGISTER, + .selector_mask = 0xfff, .selector_shift = 0, - .window_start = 0x0, - .window_len = 0x1000, + .window_start = 0x800, + .window_len = 0x400, }, }; @@ -4981,7 +4981,7 @@ static struct regmap_config wcd9335_ifc_regmap_config = { .reg_bits = 16, .val_bits = 8, .can_multi_write = true, - .max_register = WCD9335_REG(0, 0x7FF), + .max_register = WCD9335_MAX_REGISTER, .ranges = wcd9335_ifc_ranges, .num_ranges = ARRAY_SIZE(wcd9335_ifc_ranges), }; diff --git a/sound/soc/codecs/wcd9335.h b/sound/soc/codecs/wcd9335.h index 4d9be2496c30..72060824c743 100644 --- a/sound/soc/codecs/wcd9335.h +++ b/sound/soc/codecs/wcd9335.h @@ -8,9 +8,9 @@ * in slimbus mode the reg base starts from 0x800 * in i2s/i2c mode the reg base is 0x0 */ -#define WCD9335_REG(pg, r) ((pg << 12) | (r) | 0x800) +#define WCD9335_REG(pg, r) ((pg << 8) | (r)) #define WCD9335_REG_OFFSET(r) (r & 0xFF) -#define WCD9335_PAGE_OFFSET(r) ((r >> 12) & 0xFF) +#define WCD9335_PAGE_OFFSET(r) ((r >> 8) & 0xFF) /* Page-0 Registers */ #define WCD9335_PAGE0_PAGE_REGISTER WCD9335_REG(0x00, 0x000) @@ -600,7 +600,8 @@ #define WCD9335_CDC_CLK_RST_CTRL_FS_CNT_ENABLE BIT(0) #define WCD9335_CDC_CLK_RST_CTRL_FS_CNT_DISABLE 0 #define WCD9335_CDC_TOP_TOP_CFG1 WCD9335_REG(0x0d, 0x082) -#define WCD9335_MAX_REGISTER WCD9335_REG(0x80, 0x0FF) +#define WCD9335_MAX_REGISTER 0xffff +#define WCD9335_SEL_REGISTER 0x800 /* SLIMBUS Slave Registers */ #define WCD9335_SLIM_PGD_PORT_INT_EN0 WCD9335_REG(0, 0x30) diff --git a/sound/soc/codecs/wcd934x.c b/sound/soc/codecs/wcd934x.c index 158e878abd6c..5269857e2746 100644 --- a/sound/soc/codecs/wcd934x.c +++ b/sound/soc/codecs/wcd934x.c @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -11,10 +10,7 @@ #include #include #include -#include -#include #include -#include #include #include #include @@ -1202,11 +1198,6 @@ static int wcd934x_set_sido_input_src(struct wcd934x_codec *wcd, int sido_src) regmap_update_bits(wcd->regmap, WCD934X_ANA_RCO, WCD934X_ANA_RCO_BG_EN_MASK, 0); usleep_range(100, 110); - } else if (sido_src == SIDO_SOURCE_RCO_BG) { - regmap_update_bits(wcd->regmap, WCD934X_ANA_RCO, - WCD934X_ANA_RCO_BG_EN_MASK, - WCD934X_ANA_RCO_BG_ENABLE); - usleep_range(100, 110); regmap_update_bits(wcd->regmap, WCD934X_ANA_BUCK_CTL, WCD934X_ANA_BUCK_PRE_EN1_MASK, WCD934X_ANA_BUCK_PRE_EN1_ENABLE); @@ -1219,6 +1210,11 @@ static int wcd934x_set_sido_input_src(struct wcd934x_codec *wcd, int sido_src) WCD934X_ANA_BUCK_HI_ACCU_EN_MASK, WCD934X_ANA_BUCK_HI_ACCU_ENABLE); usleep_range(100, 110); + } else if (sido_src == SIDO_SOURCE_RCO_BG) { + regmap_update_bits(wcd->regmap, WCD934X_ANA_RCO, + WCD934X_ANA_RCO_BG_EN_MASK, + WCD934X_ANA_RCO_BG_ENABLE); + usleep_range(100, 110); } wcd->sido_input_src = sido_src; @@ -1883,20 +1879,16 @@ static int wcd934x_set_channel_map(struct snd_soc_dai *dai, return -EINVAL; } - if (wcd->rx_chs) { - wcd->num_rx_port = rx_num; - for (i = 0; i < rx_num; i++) { - wcd->rx_chs[i].ch_num = rx_slot[i]; - INIT_LIST_HEAD(&wcd->rx_chs[i].list); - } + wcd->num_rx_port = rx_num; + for (i = 0; i < rx_num; i++) { + wcd->rx_chs[i].ch_num = rx_slot[i]; + INIT_LIST_HEAD(&wcd->rx_chs[i].list); } - if (wcd->tx_chs) { - wcd->num_tx_port = tx_num; - for (i = 0; i < tx_num; i++) { - wcd->tx_chs[i].ch_num = tx_slot[i]; - INIT_LIST_HEAD(&wcd->tx_chs[i].list); - } + wcd->num_tx_port = tx_num; + for (i = 0; i < tx_num; i++) { + wcd->tx_chs[i].ch_num = tx_slot[i]; + INIT_LIST_HEAD(&wcd->tx_chs[i].list); } return 0; @@ -3392,18 +3384,15 @@ static void wcd934x_codec_hphdelay_lutbypass(struct snd_soc_component *comp, { u8 hph_dly_mask; u16 hph_lut_bypass_reg = 0; - u16 hph_comp_ctrl7 = 0; switch (interp_idx) { case INTERP_HPHL: hph_dly_mask = 1; hph_lut_bypass_reg = WCD934X_CDC_TOP_HPHL_COMP_LUT; - hph_comp_ctrl7 = WCD934X_CDC_COMPANDER1_CTL7; break; case INTERP_HPHR: hph_dly_mask = 2; hph_lut_bypass_reg = WCD934X_CDC_TOP_HPHR_COMP_LUT; - hph_comp_ctrl7 = WCD934X_CDC_COMPANDER2_CTL7; break; default: return; diff --git a/sound/soc/codecs/wm0010.c b/sound/soc/codecs/wm0010.c index 727d6703c905..fbcee21736e8 100644 --- a/sound/soc/codecs/wm0010.c +++ b/sound/soc/codecs/wm0010.c @@ -43,7 +43,7 @@ struct dfw_binrec { u8 command; u32 length:24; u32 address; - uint8_t data[0]; + uint8_t data[]; } __packed; struct dfw_inforec { diff --git a/sound/soc/codecs/wm5110.c b/sound/soc/codecs/wm5110.c index 9dc215b5c504..499e87d1dfcc 100644 --- a/sound/soc/codecs/wm5110.c +++ b/sound/soc/codecs/wm5110.c @@ -2245,14 +2245,14 @@ static int wm5110_open(struct snd_compr_stream *stream) struct arizona *arizona = priv->core.arizona; int n_adsp; - if (strcmp(rtd->codec_dai->name, "wm5110-dsp-voicectrl") == 0) { + if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "wm5110-dsp-voicectrl") == 0) { n_adsp = 2; - } else if (strcmp(rtd->codec_dai->name, "wm5110-dsp-trace") == 0) { + } else if (strcmp(asoc_rtd_to_codec(rtd, 0)->name, "wm5110-dsp-trace") == 0) { n_adsp = 0; } else { dev_err(arizona->dev, "No suitable compressed stream for DAI '%s'\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); return -EINVAL; } diff --git a/sound/soc/codecs/wm8974.c b/sound/soc/codecs/wm8974.c index dc4fe4f5239d..06ba36595ddd 100644 --- a/sound/soc/codecs/wm8974.c +++ b/sound/soc/codecs/wm8974.c @@ -196,14 +196,6 @@ SOC_DAPM_SINGLE("MicN Switch", WM8974_INPUT, 1, 1, 0), SOC_DAPM_SINGLE("MicP Switch", WM8974_INPUT, 0, 1, 0), }; -/* AUX Input boost vol */ -static const struct snd_kcontrol_new wm8974_aux_boost_controls = -SOC_DAPM_SINGLE("Aux Volume", WM8974_ADCBOOST, 0, 7, 0); - -/* Mic Input boost vol */ -static const struct snd_kcontrol_new wm8974_mic_boost_controls = -SOC_DAPM_SINGLE("Mic Volume", WM8974_ADCBOOST, 4, 7, 0); - static const struct snd_soc_dapm_widget wm8974_dapm_widgets[] = { SND_SOC_DAPM_MIXER("Speaker Mixer", WM8974_POWER3, 2, 0, &wm8974_speaker_mixer_controls[0], diff --git a/sound/soc/codecs/wm_adsp.c b/sound/soc/codecs/wm_adsp.c index d3d32b501aca..1ef69409ccd1 100644 --- a/sound/soc/codecs/wm_adsp.c +++ b/sound/soc/codecs/wm_adsp.c @@ -1436,12 +1436,12 @@ static int wm_adsp_create_control(struct wm_adsp *dsp, subname = NULL; /* don't append subname */ break; case 2: - ret = snprintf(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, + ret = scnprintf(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s%c %.12s %x", dsp->name, *region_name, wm_adsp_fw_text[dsp->fw], alg_region->alg); break; default: - ret = snprintf(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, + ret = scnprintf(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s %.12s %x", dsp->name, wm_adsp_fw_text[dsp->fw], alg_region->alg); break; @@ -3467,22 +3467,22 @@ int wm_adsp_compr_open(struct wm_adsp *dsp, struct snd_compr_stream *stream) if (wm_adsp_fw[dsp->fw].num_caps == 0) { adsp_err(dsp, "%s: Firmware does not support compressed API\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); ret = -ENXIO; goto out; } if (wm_adsp_fw[dsp->fw].compr_direction != stream->direction) { adsp_err(dsp, "%s: Firmware does not support stream direction\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); ret = -EINVAL; goto out; } list_for_each_entry(tmp, &dsp->compr_list, list) { - if (!strcmp(tmp->name, rtd->codec_dai->name)) { + if (!strcmp(tmp->name, asoc_rtd_to_codec(rtd, 0)->name)) { adsp_err(dsp, "%s: Only a single stream supported per dai\n", - rtd->codec_dai->name); + asoc_rtd_to_codec(rtd, 0)->name); ret = -EBUSY; goto out; } @@ -3496,7 +3496,7 @@ int wm_adsp_compr_open(struct wm_adsp *dsp, struct snd_compr_stream *stream) compr->dsp = dsp; compr->stream = stream; - compr->name = rtd->codec_dai->name; + compr->name = asoc_rtd_to_codec(rtd, 0)->name; list_add_tail(&compr->list, &dsp->compr_list); diff --git a/sound/soc/codecs/wsa881x.c b/sound/soc/codecs/wsa881x.c index b59f1d0e7f84..f2d6f2f81f14 100644 --- a/sound/soc/codecs/wsa881x.c +++ b/sound/soc/codecs/wsa881x.c @@ -676,7 +676,6 @@ struct wsa881x_priv { int active_ports; bool port_prepared[WSA881X_MAX_SWR_PORTS]; bool port_enable[WSA881X_MAX_SWR_PORTS]; - bool stream_prepared; }; static void wsa881x_init(struct wsa881x_priv *wsa881x) @@ -954,41 +953,6 @@ static const struct snd_soc_dapm_widget wsa881x_dapm_widgets[] = { SND_SOC_DAPM_OUTPUT("SPKR"), }; -static int wsa881x_prepare(struct snd_pcm_substream *substream, - struct snd_soc_dai *dai) -{ - struct wsa881x_priv *wsa881x = dev_get_drvdata(dai->dev); - int ret; - - if (wsa881x->stream_prepared) { - sdw_disable_stream(wsa881x->sruntime); - sdw_deprepare_stream(wsa881x->sruntime); - wsa881x->stream_prepared = false; - } - - - ret = sdw_prepare_stream(wsa881x->sruntime); - if (ret) - return ret; - - /** - * NOTE: there is a strict hw requirement about the ordering of port - * enables and actual PA enable. PA enable should only happen after - * soundwire ports are enabled if not DC on the line is accumulated - * resulting in Click/Pop Noise - * PA enable/mute are handled as part of DAPM and digital mute. - */ - - ret = sdw_enable_stream(wsa881x->sruntime); - if (ret) { - sdw_deprepare_stream(wsa881x->sruntime); - return ret; - } - wsa881x->stream_prepared = true; - - return ret; -} - static int wsa881x_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) @@ -1016,12 +980,7 @@ static int wsa881x_hw_free(struct snd_pcm_substream *substream, { struct wsa881x_priv *wsa881x = dev_get_drvdata(dai->dev); - if (wsa881x->stream_prepared) { - sdw_disable_stream(wsa881x->sruntime); - sdw_deprepare_stream(wsa881x->sruntime); - sdw_stream_remove_slave(wsa881x->slave, wsa881x->sruntime); - wsa881x->stream_prepared = false; - } + sdw_stream_remove_slave(wsa881x->slave, wsa881x->sruntime); return 0; } @@ -1052,7 +1011,6 @@ static int wsa881x_digital_mute(struct snd_soc_dai *dai, int mute, int stream) static struct snd_soc_dai_ops wsa881x_dai_ops = { .hw_params = wsa881x_hw_params, - .prepare = wsa881x_prepare, .hw_free = wsa881x_hw_free, .mute_stream = wsa881x_digital_mute, .set_sdw_stream = wsa881x_set_sdw_stream, @@ -1150,7 +1108,7 @@ static int wsa881x_probe(struct sdw_slave *pdev, wsa881x->sconfig.type = SDW_STREAM_PDM; pdev->prop.sink_ports = GENMASK(WSA881X_MAX_SWR_PORTS, 0); pdev->prop.sink_dpn_prop = wsa_sink_dpn_prop; - gpiod_set_value(wsa881x->sd_n, 1); + gpiod_direction_output(wsa881x->sd_n, 1); wsa881x->regmap = devm_regmap_init_sdw(pdev, &wsa881x_regmap_config); if (IS_ERR(wsa881x->regmap)) { diff --git a/sound/soc/dwc/dwc-i2s.c b/sound/soc/dwc/dwc-i2s.c index 7eeca2150b2d..515f88456dbd 100644 --- a/sound/soc/dwc/dwc-i2s.c +++ b/sound/soc/dwc/dwc-i2s.c @@ -422,15 +422,15 @@ static int dw_i2s_resume(struct snd_soc_component *component) { struct dw_i2s_dev *dev = snd_soc_component_get_drvdata(component); struct snd_soc_dai *dai; + int stream; if (dev->capability & DW_I2S_MASTER) clk_enable(dev->clk); for_each_component_dais(component, dai) { - if (dai->playback_active) - dw_i2s_config(dev, SNDRV_PCM_STREAM_PLAYBACK); - if (dai->capture_active) - dw_i2s_config(dev, SNDRV_PCM_STREAM_CAPTURE); + for_each_pcm_streams(stream) + if (dai->stream_active[stream]) + dw_i2s_config(dev, stream); } return 0; diff --git a/sound/soc/dwc/dwc-pcm.c b/sound/soc/dwc/dwc-pcm.c index 4b25aca3804f..9868e7373d36 100644 --- a/sound/soc/dwc/dwc-pcm.c +++ b/sound/soc/dwc/dwc-pcm.c @@ -140,7 +140,7 @@ static int dw_pcm_open(struct snd_soc_component *component, { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct dw_i2s_dev *dev = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct dw_i2s_dev *dev = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); snd_soc_set_runtime_hwparams(substream, &dw_pcm_hardware); snd_pcm_hw_constraint_integer(runtime, SNDRV_PCM_HW_PARAM_PERIODS); diff --git a/sound/soc/fsl/eukrea-tlv320.c b/sound/soc/fsl/eukrea-tlv320.c index 6f3b768489f6..4ff2d21bb32f 100644 --- a/sound/soc/fsl/eukrea-tlv320.c +++ b/sound/soc/fsl/eukrea-tlv320.c @@ -31,8 +31,8 @@ static int eukrea_tlv320_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, 0, diff --git a/sound/soc/fsl/fsl-asoc-card.c b/sound/soc/fsl/fsl-asoc-card.c index 9ce55feaac22..bb33601fab84 100644 --- a/sound/soc/fsl/fsl-asoc-card.c +++ b/sound/soc/fsl/fsl-asoc-card.c @@ -159,7 +159,7 @@ static int fsl_asoc_card_hw_params(struct snd_pcm_substream *substream, return 0; /* Specific configurations of DAIs starts from here */ - ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, cpu_priv->sysclk_id[tx], + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(rtd, 0), cpu_priv->sysclk_id[tx], cpu_priv->sysclk_freq[tx], cpu_priv->sysclk_dir[tx]); if (ret && ret != -ENOTSUPP) { @@ -168,7 +168,7 @@ static int fsl_asoc_card_hw_params(struct snd_pcm_substream *substream, } if (cpu_priv->slot_width) { - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), 0x3, 0x3, 2, cpu_priv->slot_width); if (ret && ret != -ENOTSUPP) { dev_err(dev, "failed to set TDM slot for cpu dai\n"); @@ -257,7 +257,7 @@ static int fsl_asoc_card_set_bias_level(struct snd_soc_card *card, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - codec_dai = rtd->codec_dai; + codec_dai = asoc_rtd_to_codec(rtd, 0); if (dapm->dev != codec_dai->dev) return 0; @@ -446,14 +446,14 @@ static int fsl_asoc_card_late_probe(struct snd_soc_card *card) struct fsl_asoc_card_priv *priv = snd_soc_card_get_drvdata(card); struct snd_soc_pcm_runtime *rtd = list_first_entry( &card->rtd_list, struct snd_soc_pcm_runtime, list); - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct codec_priv *codec_priv = &priv->codec_priv; struct device *dev = card->dev; int ret; if (fsl_asoc_card_is_ac97(priv)) { #if IS_ENABLED(CONFIG_SND_AC97_CODEC) - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct snd_ac97 *ac97 = snd_soc_component_get_drvdata(component); /* diff --git a/sound/soc/fsl/fsl_asrc_dma.c b/sound/soc/fsl/fsl_asrc_dma.c index ece130f59d15..e7178817d7a7 100644 --- a/sound/soc/fsl/fsl_asrc_dma.c +++ b/sound/soc/fsl/fsl_asrc_dma.c @@ -152,7 +152,7 @@ static int fsl_asrc_dma_hw_params(struct snd_soc_component *component, for_each_dpcm_be(rtd, stream, dpcm) { struct snd_soc_pcm_runtime *be = dpcm->be; struct snd_pcm_substream *substream_be; - struct snd_soc_dai *dai = be->cpu_dai; + struct snd_soc_dai *dai = asoc_rtd_to_cpu(be, 0); if (dpcm->fe != rtd) continue; @@ -169,7 +169,7 @@ static int fsl_asrc_dma_hw_params(struct snd_soc_component *component, } /* Override dma_data of the Front-End and config its dmaengine */ - dma_params_fe = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + dma_params_fe = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); dma_params_fe->addr = asrc_priv->paddr + REG_ASRDx(!dir, index); dma_params_fe->maxburst = dma_params_be->maxburst; @@ -328,7 +328,7 @@ static int fsl_asrc_dma_startup(struct snd_soc_component *component, goto dma_chan_err; } - dma_data = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + dma_data = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); /* Refine the snd_imx_hardware according to caps of DMA. */ ret = snd_dmaengine_pcm_refine_runtime_hwparams(substream, @@ -400,7 +400,7 @@ static int fsl_asrc_dma_pcm_new(struct snd_soc_component *component, return ret; } - for (i = SNDRV_PCM_STREAM_PLAYBACK; i <= SNDRV_PCM_STREAM_LAST; i++) { + for_each_pcm_streams(i) { substream = pcm->streams[i].substream; if (!substream) continue; @@ -428,7 +428,7 @@ static void fsl_asrc_dma_pcm_free(struct snd_soc_component *component, struct snd_pcm_substream *substream; int i; - for (i = SNDRV_PCM_STREAM_PLAYBACK; i <= SNDRV_PCM_STREAM_LAST; i++) { + for_each_pcm_streams(i) { substream = pcm->streams[i].substream; if (!substream) continue; diff --git a/sound/soc/fsl/fsl_spdif.c b/sound/soc/fsl/fsl_spdif.c index 7858a5499ac5..c711d2d93280 100644 --- a/sound/soc/fsl/fsl_spdif.c +++ b/sound/soc/fsl/fsl_spdif.c @@ -370,7 +370,7 @@ static int spdif_set_sample_rate(struct snd_pcm_substream *substream, int sample_rate) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct spdif_mixer_control *ctrl = &spdif_priv->fsl_spdif_control; struct regmap *regmap = spdif_priv->regmap; struct platform_device *pdev = spdif_priv->pdev; @@ -458,7 +458,7 @@ static int fsl_spdif_startup(struct snd_pcm_substream *substream, struct snd_soc_dai *cpu_dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct platform_device *pdev = spdif_priv->pdev; struct regmap *regmap = spdif_priv->regmap; u32 scr, mask; @@ -534,7 +534,7 @@ static void fsl_spdif_shutdown(struct snd_pcm_substream *substream, struct snd_soc_dai *cpu_dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct regmap *regmap = spdif_priv->regmap; u32 scr, mask, i; @@ -569,7 +569,7 @@ static int fsl_spdif_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct spdif_mixer_control *ctrl = &spdif_priv->fsl_spdif_control; struct platform_device *pdev = spdif_priv->pdev; u32 sample_rate = params_rate(params); @@ -597,7 +597,7 @@ static int fsl_spdif_trigger(struct snd_pcm_substream *substream, int cmd, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct regmap *regmap = spdif_priv->regmap; bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; u32 intr = SIE_INTR_FOR(tx); diff --git a/sound/soc/fsl/fsl_ssi.c b/sound/soc/fsl/fsl_ssi.c index 5c97269be346..bad89b0d129e 100644 --- a/sound/soc/fsl/fsl_ssi.c +++ b/sound/soc/fsl/fsl_ssi.c @@ -631,7 +631,7 @@ static int fsl_ssi_startup(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct fsl_ssi *ssi = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct fsl_ssi *ssi = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); int ret; ret = clk_prepare_enable(ssi->clk); @@ -655,7 +655,7 @@ static void fsl_ssi_shutdown(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct fsl_ssi *ssi = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct fsl_ssi *ssi = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); clk_disable_unprepare(ssi->clk); } @@ -854,7 +854,7 @@ static int fsl_ssi_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct fsl_ssi *ssi = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct fsl_ssi *ssi = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); if (fsl_ssi_is_i2s_master(ssi) && ssi->baudclk_streams & BIT(substream->stream)) { @@ -1059,7 +1059,7 @@ static int fsl_ssi_trigger(struct snd_pcm_substream *substream, int cmd, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct fsl_ssi *ssi = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct fsl_ssi *ssi = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; switch (cmd) { diff --git a/sound/soc/fsl/imx-audmix.c b/sound/soc/fsl/imx-audmix.c index 5ef6881395e0..e09b45de0efd 100644 --- a/sound/soc/fsl/imx-audmix.c +++ b/sound/soc/fsl/imx-audmix.c @@ -85,13 +85,13 @@ static int imx_audmix_fe_hw_params(struct snd_pcm_substream *substream, dir = tx ? SND_SOC_CLOCK_OUT : SND_SOC_CLOCK_IN; /* set DAI configuration */ - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, fmt); + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), fmt); if (ret) { dev_err(dev, "failed to set cpu dai fmt: %d\n", ret); return ret; } - ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, FSL_SAI_CLK_MAST1, 0, dir); + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(rtd, 0), FSL_SAI_CLK_MAST1, 0, dir); if (ret) { dev_err(dev, "failed to set cpu sysclk: %d\n", ret); return ret; @@ -101,7 +101,7 @@ static int imx_audmix_fe_hw_params(struct snd_pcm_substream *substream, * Per datasheet, AUDMIX expects 8 slots and 32 bits * for every slot in TDM mode. */ - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, BIT(channels) - 1, + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), BIT(channels) - 1, BIT(channels) - 1, 8, 32); if (ret) dev_err(dev, "failed to set cpu dai tdm slot: %d\n", ret); @@ -125,7 +125,7 @@ static int imx_audmix_be_hw_params(struct snd_pcm_substream *substream, fmt |= SND_SOC_DAIFMT_CBM_CFM; /* set AUDMIX DAI configuration */ - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, fmt); + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), fmt); if (ret) dev_err(dev, "failed to set AUDMIX DAI fmt: %d\n", ret); diff --git a/sound/soc/fsl/imx-mc13783.c b/sound/soc/fsl/imx-mc13783.c index 2b679680c93f..fab2d6c56653 100644 --- a/sound/soc/fsl/imx-mc13783.c +++ b/sound/soc/fsl/imx-mc13783.c @@ -27,8 +27,8 @@ static int imx_mc13783_hifi_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_tdm_slot(codec_dai, 0x3, 0x3, 4, 16); diff --git a/sound/soc/fsl/imx-sgtl5000.c b/sound/soc/fsl/imx-sgtl5000.c index 15e8b9343c35..f45cb4bbb6c4 100644 --- a/sound/soc/fsl/imx-sgtl5000.c +++ b/sound/soc/fsl/imx-sgtl5000.c @@ -30,7 +30,7 @@ static int imx_sgtl5000_dai_init(struct snd_soc_pcm_runtime *rtd) struct device *dev = rtd->card->dev; int ret; - ret = snd_soc_dai_set_sysclk(rtd->codec_dai, SGTL5000_SYSCLK, + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_codec(rtd, 0), SGTL5000_SYSCLK, data->clk_frequency, SND_SOC_CLOCK_IN); if (ret) { dev_err(dev, "could not set codec driver clock params\n"); diff --git a/sound/soc/fsl/mpc5200_dma.c b/sound/soc/fsl/mpc5200_dma.c index ed7211d744b3..3b8c796d7829 100644 --- a/sound/soc/fsl/mpc5200_dma.c +++ b/sound/soc/fsl/mpc5200_dma.c @@ -115,7 +115,7 @@ static int psc_dma_trigger(struct snd_soc_component *component, struct snd_pcm_substream *substream, int cmd) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct snd_pcm_runtime *runtime = substream->runtime; struct psc_dma_stream *s = to_psc_dma_stream(substream, psc_dma); struct mpc52xx_psc __iomem *regs = psc_dma->psc_regs; @@ -217,7 +217,7 @@ static int psc_dma_open(struct snd_soc_component *component, { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct psc_dma_stream *s; int rc; @@ -245,7 +245,7 @@ static int psc_dma_close(struct snd_soc_component *component, struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct psc_dma_stream *s; dev_dbg(psc_dma->dev, "psc_dma_close(substream=%p)\n", substream); @@ -271,7 +271,7 @@ psc_dma_pointer(struct snd_soc_component *component, struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct psc_dma_stream *s; dma_addr_t count; @@ -298,7 +298,7 @@ static int psc_dma_new(struct snd_soc_component *component, struct snd_soc_pcm_runtime *rtd) { struct snd_card *card = rtd->card->snd_card; - struct snd_soc_dai *dai = rtd->cpu_dai; + struct snd_soc_dai *dai = asoc_rtd_to_cpu(rtd, 0); struct snd_pcm *pcm = rtd->pcm; size_t size = psc_dma_hardware.buffer_bytes_max; int rc; diff --git a/sound/soc/fsl/mpc5200_psc_i2s.c b/sound/soc/fsl/mpc5200_psc_i2s.c index 9bc01f374b39..1ab4fbda08cb 100644 --- a/sound/soc/fsl/mpc5200_psc_i2s.c +++ b/sound/soc/fsl/mpc5200_psc_i2s.c @@ -39,7 +39,7 @@ static int psc_i2s_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct psc_dma *psc_dma = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); u32 mode; dev_dbg(psc_dma->dev, "%s(substream=%p) p_size=%i p_bytes=%i" diff --git a/sound/soc/fsl/mpc8610_hpcd.c b/sound/soc/fsl/mpc8610_hpcd.c index 23617eb09ba1..f7bd90051ce7 100644 --- a/sound/soc/fsl/mpc8610_hpcd.c +++ b/sound/soc/fsl/mpc8610_hpcd.c @@ -105,7 +105,7 @@ static int mpc8610_hpcd_startup(struct snd_pcm_substream *substream) int ret = 0; /* Tell the codec driver what the serial protocol is. */ - ret = snd_soc_dai_set_fmt(rtd->codec_dai, machine_data->dai_format); + ret = snd_soc_dai_set_fmt(asoc_rtd_to_codec(rtd, 0), machine_data->dai_format); if (ret < 0) { dev_err(dev, "could not set codec driver audio format\n"); return ret; @@ -115,7 +115,7 @@ static int mpc8610_hpcd_startup(struct snd_pcm_substream *substream) * Tell the codec driver what the MCLK frequency is, and whether it's * a slave or master. */ - ret = snd_soc_dai_set_sysclk(rtd->codec_dai, 0, + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_codec(rtd, 0), 0, machine_data->clk_frequency, machine_data->codec_clk_direction); if (ret < 0) { diff --git a/sound/soc/fsl/mx27vis-aic32x4.c b/sound/soc/fsl/mx27vis-aic32x4.c index 38ac4a397742..a36d4e8cd55c 100644 --- a/sound/soc/fsl/mx27vis-aic32x4.c +++ b/sound/soc/fsl/mx27vis-aic32x4.c @@ -37,8 +37,8 @@ static int mx27vis_aic32x4_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, 0, diff --git a/sound/soc/fsl/p1022_ds.c b/sound/soc/fsl/p1022_ds.c index 6114b01b90f7..fe3091590f20 100644 --- a/sound/soc/fsl/p1022_ds.c +++ b/sound/soc/fsl/p1022_ds.c @@ -128,7 +128,7 @@ static int p1022_ds_startup(struct snd_pcm_substream *substream) int ret = 0; /* Tell the codec driver what the serial protocol is. */ - ret = snd_soc_dai_set_fmt(rtd->codec_dai, mdata->dai_format); + ret = snd_soc_dai_set_fmt(asoc_rtd_to_codec(rtd, 0), mdata->dai_format); if (ret < 0) { dev_err(dev, "could not set codec driver audio format\n"); return ret; @@ -138,7 +138,7 @@ static int p1022_ds_startup(struct snd_pcm_substream *substream) * Tell the codec driver what the MCLK frequency is, and whether it's * a slave or master. */ - ret = snd_soc_dai_set_sysclk(rtd->codec_dai, 0, mdata->clk_frequency, + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_codec(rtd, 0), 0, mdata->clk_frequency, mdata->codec_clk_direction); if (ret < 0) { dev_err(dev, "could not set codec driver clock params\n"); diff --git a/sound/soc/fsl/p1022_rdk.c b/sound/soc/fsl/p1022_rdk.c index 72687235c0ae..f5374fe354ab 100644 --- a/sound/soc/fsl/p1022_rdk.c +++ b/sound/soc/fsl/p1022_rdk.c @@ -134,14 +134,14 @@ static int p1022_rdk_startup(struct snd_pcm_substream *substream) int ret = 0; /* Tell the codec driver what the serial protocol is. */ - ret = snd_soc_dai_set_fmt(rtd->codec_dai, mdata->dai_format); + ret = snd_soc_dai_set_fmt(asoc_rtd_to_codec(rtd, 0), mdata->dai_format); if (ret < 0) { dev_err(dev, "could not set codec driver audio format (ret=%i)\n", ret); return ret; } - ret = snd_soc_dai_set_pll(rtd->codec_dai, 0, 0, mdata->clk_frequency, + ret = snd_soc_dai_set_pll(asoc_rtd_to_codec(rtd, 0), 0, 0, mdata->clk_frequency, mdata->clk_frequency); if (ret < 0) { dev_err(dev, "could not set codec PLL frequency (ret=%i)\n", diff --git a/sound/soc/fsl/wm1133-ev1.c b/sound/soc/fsl/wm1133-ev1.c index 52d321bede9c..8b1551c55452 100644 --- a/sound/soc/fsl/wm1133-ev1.c +++ b/sound/soc/fsl/wm1133-ev1.c @@ -76,8 +76,8 @@ static int wm1133_ev1_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int i, found = 0; snd_pcm_format_t format = params_format(params); unsigned int rate = params_rate(params); @@ -196,7 +196,7 @@ static struct snd_soc_jack_pin mic_jack_pins[] = { static int wm1133_ev1_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; /* Headphone jack detection */ snd_soc_card_jack_new(rtd->card, "Headphone", SND_JACK_HEADPHONE, diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c index 9b794775df53..8c54dc6710fe 100644 --- a/sound/soc/generic/simple-card-utils.c +++ b/sound/soc/generic/simple-card-utils.c @@ -213,8 +213,8 @@ EXPORT_SYMBOL_GPL(asoc_simple_startup); void asoc_simple_shutdown(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(rtd->card); struct simple_dai_props *dai_props = simple_priv_to_props(priv, rtd->num); @@ -249,8 +249,8 @@ int asoc_simple_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(rtd->card); struct simple_dai_props *dai_props = simple_priv_to_props(priv, rtd->num); @@ -331,22 +331,70 @@ static int asoc_simple_init_dai(struct snd_soc_dai *dai, return 0; } +static int asoc_simple_init_dai_link_params(struct snd_soc_pcm_runtime *rtd, + struct simple_dai_props *dai_props) +{ + struct snd_soc_dai_link *dai_link = rtd->dai_link; + struct snd_soc_component *component; + struct snd_soc_pcm_stream *params; + struct snd_pcm_hardware hw; + int i, ret, stream; + + /* Only codecs should have non_legacy_dai_naming set. */ + for_each_rtd_components(rtd, i, component) { + if (!component->driver->non_legacy_dai_naming) + return 0; + } + + /* Assumes the capabilities are the same for all supported streams */ + for_each_pcm_streams(stream) { + ret = snd_soc_runtime_calc_hw(rtd, &hw, stream); + if (ret == 0) + break; + } + + if (ret < 0) { + dev_err(rtd->dev, "simple-card: no valid dai_link params\n"); + return ret; + } + + params = devm_kzalloc(rtd->dev, sizeof(*params), GFP_KERNEL); + if (!params) + return -ENOMEM; + + params->formats = hw.formats; + params->rates = hw.rates; + params->rate_min = hw.rate_min; + params->rate_max = hw.rate_max; + params->channels_min = hw.channels_min; + params->channels_max = hw.channels_max; + + dai_link->params = params; + dai_link->num_params = 1; + + return 0; +} + int asoc_simple_dai_init(struct snd_soc_pcm_runtime *rtd) { struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(rtd->card); struct simple_dai_props *dai_props = simple_priv_to_props(priv, rtd->num); int ret; - ret = asoc_simple_init_dai(rtd->codec_dai, + ret = asoc_simple_init_dai(asoc_rtd_to_codec(rtd, 0), dai_props->codec_dai); if (ret < 0) return ret; - ret = asoc_simple_init_dai(rtd->cpu_dai, + ret = asoc_simple_init_dai(asoc_rtd_to_cpu(rtd, 0), dai_props->cpu_dai); if (ret < 0) return ret; + ret = asoc_simple_init_dai_link_params(rtd, dai_props); + if (ret < 0) + return ret; + return 0; } EXPORT_SYMBOL_GPL(asoc_simple_dai_init); diff --git a/sound/soc/img/img-i2s-in.c b/sound/soc/img/img-i2s-in.c index fdd2c73fd2fa..a495d1050d49 100644 --- a/sound/soc/img/img-i2s-in.c +++ b/sound/soc/img/img-i2s-in.c @@ -397,7 +397,7 @@ static int img_i2s_in_dma_prepare_slave_config(struct snd_pcm_substream *st, struct snd_dmaengine_dai_dma_data *dma_data; int ret; - dma_data = snd_soc_dai_get_dma_data(rtd->cpu_dai, st); + dma_data = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), st); ret = snd_hwparams_to_dma_slave_config(st, params, sc); if (ret) diff --git a/sound/soc/img/img-i2s-out.c b/sound/soc/img/img-i2s-out.c index 4b1853409633..db052ec17d5d 100644 --- a/sound/soc/img/img-i2s-out.c +++ b/sound/soc/img/img-i2s-out.c @@ -403,7 +403,7 @@ static int img_i2s_out_dma_prepare_slave_config(struct snd_pcm_substream *st, struct snd_dmaengine_dai_dma_data *dma_data; int ret; - dma_data = snd_soc_dai_get_dma_data(rtd->cpu_dai, st); + dma_data = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), st); ret = snd_hwparams_to_dma_slave_config(st, params, sc); if (ret) diff --git a/sound/soc/intel/atom/sst-atom-controls.c b/sound/soc/intel/atom/sst-atom-controls.c index baef461a99f1..f883c9340eee 100644 --- a/sound/soc/intel/atom/sst-atom-controls.c +++ b/sound/soc/intel/atom/sst-atom-controls.c @@ -1333,7 +1333,7 @@ int sst_send_pipe_gains(struct snd_soc_dai *dai, int stream, int mute) dai->capture_widget->name); w = dai->capture_widget; snd_soc_dapm_widget_for_each_source_path(w, p) { - if (p->connected && !p->connected(w, p->sink)) + if (p->connected && !p->connected(w, p->source)) continue; if (p->connect && p->source->power && diff --git a/sound/soc/intel/atom/sst-mfld-platform-pcm.c b/sound/soc/intel/atom/sst-mfld-platform-pcm.c index 340bd2be39a7..82f2b6357778 100644 --- a/sound/soc/intel/atom/sst-mfld-platform-pcm.c +++ b/sound/soc/intel/atom/sst-mfld-platform-pcm.c @@ -649,7 +649,7 @@ static snd_pcm_uframes_t sst_soc_pointer(struct snd_soc_component *component, static int sst_soc_pcm_new(struct snd_soc_component *component, struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *dai = rtd->cpu_dai; + struct snd_soc_dai *dai = asoc_rtd_to_cpu(rtd, 0); struct snd_pcm *pcm = rtd->pcm; if (dai->driver->playback.channels_min || @@ -741,7 +741,7 @@ static int sst_soc_prepare(struct device *dev) /* set the SSPs to idle */ for_each_card_rtds(drv->soc_card, rtd) { - struct snd_soc_dai *dai = rtd->cpu_dai; + struct snd_soc_dai *dai = asoc_rtd_to_cpu(rtd, 0); if (dai->active) { send_ssp_cmd(dai, dai->name, 0); @@ -762,7 +762,7 @@ static void sst_soc_complete(struct device *dev) /* restart SSPs */ for_each_card_rtds(drv->soc_card, rtd) { - struct snd_soc_dai *dai = rtd->cpu_dai; + struct snd_soc_dai *dai = asoc_rtd_to_cpu(rtd, 0); if (dai->active) { sst_handle_vb_timer(dai, true); diff --git a/sound/soc/intel/atom/sst/sst_pci.c b/sound/soc/intel/atom/sst/sst_pci.c index d952719bc098..5862fe968083 100644 --- a/sound/soc/intel/atom/sst/sst_pci.c +++ b/sound/soc/intel/atom/sst/sst_pci.c @@ -99,7 +99,7 @@ static int sst_platform_get_resources(struct intel_sst_drv *ctx) dev_dbg(ctx->dev, "DRAM Ptr %p\n", ctx->dram); do_release_regions: pci_release_regions(pci); - return 0; + return ret; } /* diff --git a/sound/soc/intel/boards/Kconfig b/sound/soc/intel/boards/Kconfig index 9ca2567d0059..556c3104e641 100644 --- a/sound/soc/intel/boards/Kconfig +++ b/sound/soc/intel/boards/Kconfig @@ -289,7 +289,6 @@ config SND_SOC_INTEL_DA7219_MAX98357A_GENERIC select SND_SOC_DA7219 select SND_SOC_MAX98357A select SND_SOC_DMIC - select SND_HDA_CODEC_HDMI if SND_SOC_SOF_HDA_AUDIO_CODEC select SND_SOC_HDAC_HDMI config SND_SOC_INTEL_BXT_DA7219_MAX98357A_COMMON @@ -302,6 +301,7 @@ config SND_SOC_INTEL_BXT_DA7219_MAX98357A_MACH tristate "Broxton with DA7219 and MAX98357A in I2S Mode" depends on I2C && ACPI depends on MFD_INTEL_LPSS || COMPILE_TEST + depends on SND_HDA_CODEC_HDMI select SND_SOC_INTEL_BXT_DA7219_MAX98357A_COMMON help This adds support for ASoC machine driver for Broxton-P platforms @@ -402,6 +402,7 @@ config SND_SOC_INTEL_GLK_DA7219_MAX98357A_MACH tristate "GLK with DA7219 and MAX98357A in I2S Mode" depends on I2C && ACPI depends on MFD_INTEL_LPSS || COMPILE_TEST + depends on SND_HDA_CODEC_HDMI select SND_SOC_INTEL_BXT_DA7219_MAX98357A_COMMON help This adds support for ASoC machine driver for Geminilake platforms @@ -413,10 +414,10 @@ config SND_SOC_INTEL_GLK_RT5682_MAX98357A_MACH tristate "GLK with RT5682 and MAX98357A in I2S Mode" depends on I2C && ACPI depends on MFD_INTEL_LPSS || COMPILE_TEST + depends on SND_HDA_CODEC_HDMI select SND_SOC_RT5682 select SND_SOC_MAX98357A select SND_SOC_DMIC - select SND_HDA_CODEC_HDMI if SND_SOC_SOF_HDA_AUDIO_CODEC select SND_SOC_HDAC_HDMI help This adds support for ASoC machine driver for Geminilake platforms @@ -430,7 +431,7 @@ if SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC || SND_SOC_SOF_HDA_AUDIO_CODEC config SND_SOC_INTEL_SKL_HDA_DSP_GENERIC_MACH tristate "SKL/KBL/BXT/APL with HDA Codecs" - select SND_HDA_CODEC_HDMI if SND_SOC_SOF_HDA_AUDIO_CODEC + depends on SND_HDA_CODEC_HDMI select SND_SOC_HDAC_HDMI select SND_SOC_DMIC # SND_SOC_HDAC_HDA is already selected @@ -448,15 +449,31 @@ config SND_SOC_INTEL_SOF_RT5682_MACH depends on I2C && ACPI depends on (SND_SOC_SOF_HDA_LINK && (MFD_INTEL_LPSS || COMPILE_TEST)) ||\ (SND_SOC_SOF_BAYTRAIL && (X86_INTEL_LPSS || COMPILE_TEST)) + depends on SND_HDA_CODEC_HDMI + select SND_SOC_MAX98373 + select SND_SOC_RT1015 select SND_SOC_RT5682 select SND_SOC_DMIC - select SND_HDA_CODEC_HDMI if SND_SOC_SOF_HDA_AUDIO_CODEC select SND_SOC_HDAC_HDMI help This adds support for ASoC machine driver for SOF platforms with rt5682 codec. Say Y if you have such a device. If unsure select "N". + +config SND_SOC_INTEL_SOF_PCM512x_MACH + tristate "SOF with TI PCM512x codec" + depends on I2C && ACPI + depends on (SND_SOC_SOF_HDA_AUDIO_CODEC && (MFD_INTEL_LPSS || COMPILE_TEST)) ||\ + (SND_SOC_SOF_BAYTRAIL && (X86_INTEL_LPSS || COMPILE_TEST)) + depends on SND_HDA_CODEC_HDMI + select SND_SOC_PCM512x_I2C + help + This adds support for ASoC machine driver for SOF platforms + with TI PCM512x I2S audio codec. + Say Y or m if you have such a device. + If unsure select "N". + endif ## SND_SOC_SOF_HDA_LINK || SND_SOC_SOF_BAYTRAIL if (SND_SOC_SOF_COMETLAKE_LP && SND_SOC_SOF_HDA_LINK) @@ -476,11 +493,11 @@ config SND_SOC_INTEL_SOF_CML_RT1011_RT5682_MACH tristate "CML with RT1011 and RT5682 in I2S Mode" depends on I2C && ACPI depends on MFD_INTEL_LPSS || COMPILE_TEST + depends on SND_HDA_CODEC_HDMI select SND_SOC_RT1011 select SND_SOC_RT5682 select SND_SOC_DMIC select SND_SOC_HDAC_HDMI - select SND_HDA_CODEC_HDMI if SND_SOC_SOF_HDA_AUDIO_CODEC help This adds support for ASoC machine driver for SOF platform with RT1011 + RT5682 I2S codec. @@ -492,19 +509,43 @@ endif ## SND_SOC_SOF_COMETLAKE_LP && SND_SOC_SOF_HDA_LINK if SND_SOC_SOF_JASPERLAKE config SND_SOC_INTEL_SOF_DA7219_MAX98373_MACH - tristate "SOF with DA7219 and MAX98373 in I2S Mode" + tristate "SOF with DA7219 and MAX98373/MAX98360A in I2S Mode" depends on I2C && ACPI depends on MFD_INTEL_LPSS || COMPILE_TEST + depends on SND_HDA_CODEC_HDMI select SND_SOC_DA7219 select SND_SOC_MAX98373 select SND_SOC_DMIC - select SND_HDA_CODEC_HDMI if SND_SOC_SOF_HDA_AUDIO_CODEC help This adds support for ASoC machine driver for SOF platforms - with DA7219 + MAX98373 I2S audio codec. + with DA7219 + MAX98373/MAX98360A I2S audio codec. Say Y if you have such a device. If unsure select "N". endif ## SND_SOC_SOF_JASPERLAKE +if SND_SOC_SOF_INTEL_SOUNDWIRE + +config SND_SOC_INTEL_SOUNDWIRE_SOF_MACH + tristate "SoundWire generic machine driver" + depends on I2C && ACPI + depends on MFD_INTEL_LPSS || COMPILE_TEST + depends on SND_SOC_INTEL_USER_FRIENDLY_LONG_NAMES || COMPILE_TEST + depends on SOUNDWIRE + depends on SND_HDA_CODEC_HDMI + select SND_SOC_RT700_SDW + select SND_SOC_RT711_SDW + select SND_SOC_RT1308_SDW + select SND_SOC_RT1308 + select SND_SOC_RT715_SDW + select SND_SOC_RT5682_SDW + select SND_SOC_DMIC + help + Add support for Intel SoundWire-based platforms connected to + RT700, RT711, RT1308 and RT715 + If unsure select "N". + +endif + + endif ## SND_SOC_INTEL_MACH diff --git a/sound/soc/intel/boards/Makefile b/sound/soc/intel/boards/Makefile index b74ddd49bd39..1ef6e60bc2a0 100644 --- a/sound/soc/intel/boards/Makefile +++ b/sound/soc/intel/boards/Makefile @@ -7,6 +7,7 @@ snd-soc-sst-bdw-rt5677-mach-objs := bdw-rt5677.o snd-soc-sst-broadwell-objs := broadwell.o snd-soc-sst-bxt-da7219_max98357a-objs := bxt_da7219_max98357a.o hda_dsp_common.o snd-soc-sst-bxt-rt298-objs := bxt_rt298.o hda_dsp_common.o +snd-soc-sst-sof-pcm512x-objs := sof_pcm512x.o hda_dsp_common.o snd-soc-sst-glk-rt5682_max98357a-objs := glk_rt5682_max98357a.o hda_dsp_common.o snd-soc-sst-bytcr-rt5640-objs := bytcr_rt5640.o snd-soc-sst-bytcr-rt5651-objs := bytcr_rt5651.o @@ -18,7 +19,7 @@ snd-soc-sst-byt-cht-cx2072x-objs := bytcht_cx2072x.o snd-soc-sst-byt-cht-da7213-objs := bytcht_da7213.o snd-soc-sst-byt-cht-es8316-objs := bytcht_es8316.o snd-soc-sst-byt-cht-nocodec-objs := bytcht_nocodec.o -snd-soc-sof_rt5682-objs := sof_rt5682.o hda_dsp_common.o +snd-soc-sof_rt5682-objs := sof_rt5682.o hda_dsp_common.o sof_maxim_common.o snd-soc-cml_rt1011_rt5682-objs := cml_rt1011_rt5682.o hda_dsp_common.o snd-soc-kbl_da7219_max98357a-objs := kbl_da7219_max98357a.o snd-soc-kbl_da7219_max98927-objs := kbl_da7219_max98927.o @@ -30,13 +31,18 @@ snd-soc-skl_hda_dsp-objs := skl_hda_dsp_generic.o skl_hda_dsp_common.o hda_dsp_c snd-skl_nau88l25_max98357a-objs := skl_nau88l25_max98357a.o snd-soc-skl_nau88l25_ssm4567-objs := skl_nau88l25_ssm4567.o snd-soc-sof_da7219_max98373-objs := sof_da7219_max98373.o hda_dsp_common.o - +snd-soc-sof-sdw-objs += sof_sdw.o \ + sof_sdw_rt711.o sof_sdw_rt700.o \ + sof_sdw_rt1308.o sof_sdw_rt715.o \ + sof_sdw_rt5682.o \ + sof_sdw_dmic.o sof_sdw_hdmi.o hda_dsp_common.o obj-$(CONFIG_SND_SOC_INTEL_SOF_RT5682_MACH) += snd-soc-sof_rt5682.o obj-$(CONFIG_SND_SOC_INTEL_HASWELL_MACH) += snd-soc-sst-haswell.o obj-$(CONFIG_SND_SOC_INTEL_BYT_RT5640_MACH) += snd-soc-sst-byt-rt5640-mach.o obj-$(CONFIG_SND_SOC_INTEL_BYT_MAX98090_MACH) += snd-soc-sst-byt-max98090-mach.o obj-$(CONFIG_SND_SOC_INTEL_BXT_DA7219_MAX98357A_COMMON) += snd-soc-sst-bxt-da7219_max98357a.o obj-$(CONFIG_SND_SOC_INTEL_BXT_RT298_MACH) += snd-soc-sst-bxt-rt298.o +obj-$(CONFIG_SND_SOC_INTEL_SOF_PCM512x_MACH) += snd-soc-sst-sof-pcm512x.o obj-$(CONFIG_SND_SOC_INTEL_GLK_RT5682_MAX98357A_MACH) += snd-soc-sst-glk-rt5682_max98357a.o obj-$(CONFIG_SND_SOC_INTEL_BROADWELL_MACH) += snd-soc-sst-broadwell.o obj-$(CONFIG_SND_SOC_INTEL_BDW_RT5650_MACH) += snd-soc-sst-bdw-rt5650-mach.o @@ -62,4 +68,4 @@ obj-$(CONFIG_SND_SOC_INTEL_SKL_NAU88L25_MAX98357A_MACH) += snd-skl_nau88l25_max9 obj-$(CONFIG_SND_SOC_INTEL_SKL_NAU88L25_SSM4567_MACH) += snd-soc-skl_nau88l25_ssm4567.o obj-$(CONFIG_SND_SOC_INTEL_SKL_HDA_DSP_GENERIC_MACH) += snd-soc-skl_hda_dsp.o obj-$(CONFIG_SND_SOC_INTEL_SOF_DA7219_MAX98373_MACH) += snd-soc-sof_da7219_max98373.o - +obj-$(CONFIG_SND_SOC_INTEL_SOUNDWIRE_SOF_MACH) += snd-soc-sof-sdw.o diff --git a/sound/soc/intel/boards/bdw-rt5650.c b/sound/soc/intel/boards/bdw-rt5650.c index 1a302436d450..6c2fdb5659ed 100644 --- a/sound/soc/intel/boards/bdw-rt5650.c +++ b/sound/soc/intel/boards/bdw-rt5650.c @@ -107,7 +107,7 @@ static int bdw_rt5650_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; /* Workaround: set codec PLL to 19.2MHz that PLL source is @@ -166,8 +166,8 @@ static int bdw_rt5650_init(struct snd_soc_pcm_runtime *rtd) { struct bdw_rt5650_priv *bdw_rt5650 = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_component *component = codec_dai->component; int ret; /* Enable codec ASRC function for Stereo DAC/Stereo1 ADC/DMIC/I2S1. @@ -226,9 +226,6 @@ SND_SOC_DAILINK_DEF(be, #if IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL) SND_SOC_DAILINK_DEF(ssp0_port, DAILINK_COMP_ARRAY(COMP_CPU("ssp0-port"))); -#else -SND_SOC_DAILINK_DEF(ssp0_port, - DAILINK_COMP_ARRAY(COMP_DUMMY())); #endif static struct snd_soc_dai_link bdw_rt5650_dais[] = { @@ -264,7 +261,11 @@ static struct snd_soc_dai_link bdw_rt5650_dais[] = { .dpcm_playback = 1, .dpcm_capture = 1, .init = bdw_rt5650_init, +#if !IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL) + SND_SOC_DAILINK_REG(dummy, be, dummy), +#else SND_SOC_DAILINK_REG(ssp0_port, be, platform), +#endif }, }; @@ -298,7 +299,7 @@ static int bdw_rt5650_probe(struct platform_device *pdev) return -ENOMEM; /* override plaform name, if required */ - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; ret = snd_soc_fixup_dai_links_platform_name(&bdw_rt5650_card, mach->mach_params.platform); diff --git a/sound/soc/intel/boards/bdw-rt5677.c b/sound/soc/intel/boards/bdw-rt5677.c index bb643c99069d..6b4b64098d36 100644 --- a/sound/soc/intel/boards/bdw-rt5677.c +++ b/sound/soc/intel/boards/bdw-rt5677.c @@ -157,7 +157,7 @@ static int bdw_rt5677_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, RT5677_SCLK_S_MCLK, 24576000, @@ -174,7 +174,7 @@ static int bdw_rt5677_dsp_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, RT5677_SCLK_S_PLL1, 24576000, @@ -226,7 +226,7 @@ static int bdw_rt5677_init(struct snd_soc_pcm_runtime *rtd) { struct bdw_rt5677_priv *bdw_rt5677 = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component); int ret; @@ -298,9 +298,6 @@ SND_SOC_DAILINK_DEF(be, #if IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL) SND_SOC_DAILINK_DEF(ssp0_port, DAILINK_COMP_ARRAY(COMP_CPU("ssp0-port"))); -#else -SND_SOC_DAILINK_DEF(ssp0_port, - DAILINK_COMP_ARRAY(COMP_DUMMY())); #endif /* Wake on voice interface */ @@ -350,7 +347,11 @@ static struct snd_soc_dai_link bdw_rt5677_dais[] = { .dpcm_playback = 1, .dpcm_capture = 1, .init = bdw_rt5677_init, +#if !IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL) + SND_SOC_DAILINK_REG(dummy, be, dummy), +#else SND_SOC_DAILINK_REG(ssp0_port, be, platform), +#endif }, }; @@ -412,7 +413,7 @@ static int bdw_rt5677_probe(struct platform_device *pdev) } /* override plaform name, if required */ - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; ret = snd_soc_fixup_dai_links_platform_name(&bdw_rt5677_card, mach->mach_params.platform); if (ret) diff --git a/sound/soc/intel/boards/broadwell.c b/sound/soc/intel/boards/broadwell.c index b9c12e24c70b..acb4e36682cb 100644 --- a/sound/soc/intel/boards/broadwell.c +++ b/sound/soc/intel/boards/broadwell.c @@ -70,7 +70,7 @@ static const struct snd_soc_dapm_route broadwell_rt286_map[] = { static int broadwell_rt286_codec_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; int ret = 0; ret = snd_soc_card_jack_new(rtd->card, "Headset", SND_JACK_HEADSET | SND_JACK_BTN_0, &broadwell_headset, @@ -104,7 +104,7 @@ static int broadwell_rt286_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, RT286_SCLK_S_PLL, 24000000, @@ -167,9 +167,6 @@ SND_SOC_DAILINK_DEF(codec, #if IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL) SND_SOC_DAILINK_DEF(ssp0_port, DAILINK_COMP_ARRAY(COMP_CPU("ssp0-port"))); -#else -SND_SOC_DAILINK_DEF(ssp0_port, - DAILINK_COMP_ARRAY(COMP_DUMMY())); #endif /* broadwell digital audio interface glue - connects codec <--> CPU */ @@ -226,7 +223,11 @@ static struct snd_soc_dai_link broadwell_rt286_dais[] = { .ops = &broadwell_rt286_ops, .dpcm_playback = 1, .dpcm_capture = 1, +#if !IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL) + SND_SOC_DAILINK_REG(dummy, codec, dummy), +#else SND_SOC_DAILINK_REG(ssp0_port, codec, platform), +#endif }, }; @@ -283,7 +284,7 @@ static int broadwell_audio_probe(struct platform_device *pdev) broadwell_rt286.dev = &pdev->dev; /* override plaform name, if required */ - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; ret = snd_soc_fixup_dai_links_platform_name(&broadwell_rt286, mach->mach_params.platform); if (ret) diff --git a/sound/soc/intel/boards/bxt_da7219_max98357a.c b/sound/soc/intel/boards/bxt_da7219_max98357a.c index 9177401c37a5..44016c16f25e 100644 --- a/sound/soc/intel/boards/bxt_da7219_max98357a.c +++ b/sound/soc/intel/boards/bxt_da7219_max98357a.c @@ -179,8 +179,8 @@ static int broxton_ssp_fixup(struct snd_soc_pcm_runtime *rtd, static int broxton_da7219_codec_init(struct snd_soc_pcm_runtime *rtd) { int ret; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; int clk_freq; /* Configure sysclk for codec */ @@ -226,7 +226,7 @@ static int broxton_da7219_codec_init(struct snd_soc_pcm_runtime *rtd) static int broxton_hdmi_init(struct snd_soc_pcm_runtime *rtd) { struct bxt_card_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct bxt_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -244,7 +244,7 @@ static int broxton_hdmi_init(struct snd_soc_pcm_runtime *rtd) static int broxton_da7219_fe_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_dapm_context *dapm; - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; dapm = snd_soc_component_get_dapm(component); snd_soc_dapm_ignore_suspend(dapm, "Reference Capture"); @@ -721,7 +721,7 @@ static int broxton_audio_probe(struct platform_device *pdev) } /* override plaform name, if required */ - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; platform_name = mach->mach_params.platform; ret = snd_soc_fixup_dai_links_platform_name(&broxton_audio_card, diff --git a/sound/soc/intel/boards/bxt_rt298.c b/sound/soc/intel/boards/bxt_rt298.c index 4b67f261377c..7a4decf34191 100644 --- a/sound/soc/intel/boards/bxt_rt298.c +++ b/sound/soc/intel/boards/bxt_rt298.c @@ -155,7 +155,7 @@ static const struct snd_soc_dapm_route geminilake_rt298_map[] = { static int broxton_rt298_fe_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_dapm_context *dapm; - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; dapm = snd_soc_component_get_dapm(component); snd_soc_dapm_ignore_suspend(dapm, "Reference Capture"); @@ -165,7 +165,7 @@ static int broxton_rt298_fe_init(struct snd_soc_pcm_runtime *rtd) static int broxton_rt298_codec_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; int ret = 0; ret = snd_soc_card_jack_new(rtd->card, "Headset", @@ -186,7 +186,7 @@ static int broxton_rt298_codec_init(struct snd_soc_pcm_runtime *rtd) static int broxton_hdmi_init(struct snd_soc_pcm_runtime *rtd) { struct bxt_rt286_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct bxt_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -225,7 +225,7 @@ static int broxton_rt298_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, RT298_SCLK_S_PLL, @@ -627,7 +627,7 @@ static int broxton_audio_probe(struct platform_device *pdev) snd_soc_card_set_drvdata(card, ctx); /* override plaform name, if required */ - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; platform_name = mach->mach_params.platform; ret = snd_soc_fixup_dai_links_platform_name(card, diff --git a/sound/soc/intel/boards/byt-max98090.c b/sound/soc/intel/boards/byt-max98090.c index 01739ad75b12..f5097da28828 100644 --- a/sound/soc/intel/boards/byt-max98090.c +++ b/sound/soc/intel/boards/byt-max98090.c @@ -89,7 +89,7 @@ static int byt_max98090_init(struct snd_soc_pcm_runtime *runtime) card->dapm.idle_bias_off = true; - ret = snd_soc_dai_set_sysclk(runtime->codec_dai, + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_codec(runtime, 0), M98090_REG_SYSTEM_CLOCK, 25000000, SND_SOC_CLOCK_IN); if (ret < 0) { diff --git a/sound/soc/intel/boards/byt-rt5640.c b/sound/soc/intel/boards/byt-rt5640.c index 0c76dafdd572..ace232f8aed6 100644 --- a/sound/soc/intel/boards/byt-rt5640.c +++ b/sound/soc/intel/boards/byt-rt5640.c @@ -73,7 +73,7 @@ static int byt_rt5640_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, RT5640_SCLK_S_PLL1, @@ -123,7 +123,7 @@ static const struct dmi_system_id byt_rt5640_quirk_table[] = { static int byt_rt5640_init(struct snd_soc_pcm_runtime *runtime) { int ret; - struct snd_soc_component *component = runtime->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(runtime, 0)->component; struct snd_soc_card *card = runtime->card; const struct snd_soc_dapm_route *custom_map; int num_routes; diff --git a/sound/soc/intel/boards/bytcht_cx2072x.c b/sound/soc/intel/boards/bytcht_cx2072x.c index 67f06c95eec5..3b3df7c9008c 100644 --- a/sound/soc/intel/boards/bytcht_cx2072x.c +++ b/sound/soc/intel/boards/bytcht_cx2072x.c @@ -70,7 +70,7 @@ static const struct acpi_gpio_mapping byt_cht_cx2072x_acpi_gpios[] = { static int byt_cht_cx2072x_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_card *card = rtd->card; - struct snd_soc_component *codec = rtd->codec_dai->component; + struct snd_soc_component *codec = asoc_rtd_to_codec(rtd, 0)->component; int ret; if (devm_acpi_dev_add_driver_gpios(codec->dev, @@ -80,7 +80,7 @@ static int byt_cht_cx2072x_init(struct snd_soc_pcm_runtime *rtd) card->dapm.idle_bias_off = true; /* set the default PLL rate, the clock is handled by the codec driver */ - ret = snd_soc_dai_set_sysclk(rtd->codec_dai, CX2072X_MCLK_EXTERNAL_PLL, + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_codec(rtd, 0), CX2072X_MCLK_EXTERNAL_PLL, 19200000, SND_SOC_CLOCK_IN); if (ret) { dev_err(rtd->dev, "Could not set sysclk\n"); @@ -97,7 +97,7 @@ static int byt_cht_cx2072x_init(struct snd_soc_pcm_runtime *rtd) snd_soc_component_set_jack(codec, &byt_cht_cx2072x_headset, NULL); - snd_soc_dai_set_bclk_ratio(rtd->codec_dai, 50); + snd_soc_dai_set_bclk_ratio(asoc_rtd_to_codec(rtd, 0), 50); return ret; } @@ -123,7 +123,7 @@ static int byt_cht_cx2072x_fixup(struct snd_soc_pcm_runtime *rtd, * with explicit setting to I2S 2ch 24-bit. The word length is set with * dai_set_tdm_slot() since there is no other API exposed */ - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBS_CFS); @@ -132,7 +132,7 @@ static int byt_cht_cx2072x_fixup(struct snd_soc_pcm_runtime *rtd, return ret; } - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, 24); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), 0x3, 0x3, 2, 24); if (ret < 0) { dev_err(rtd->dev, "can't set I2S config, err %d\n", ret); return ret; diff --git a/sound/soc/intel/boards/bytcht_da7213.c b/sound/soc/intel/boards/bytcht_da7213.c index eda7a500cad6..5e96e7d02733 100644 --- a/sound/soc/intel/boards/bytcht_da7213.c +++ b/sound/soc/intel/boards/bytcht_da7213.c @@ -78,7 +78,7 @@ static int codec_fixup(struct snd_soc_pcm_runtime *rtd, * with explicit setting to I2S 2ch 24-bit. The word length is set with * dai_set_tdm_slot() since there is no other API exposed */ - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBS_CFS); @@ -87,7 +87,7 @@ static int codec_fixup(struct snd_soc_pcm_runtime *rtd, return ret; } - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, 24); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), 0x3, 0x3, 2, 24); if (ret < 0) { dev_err(rtd->dev, "can't set I2S config, err %d\n", ret); return ret; @@ -106,7 +106,7 @@ static int aif1_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, DA7213_CLKSRC_MCLK, @@ -127,7 +127,7 @@ static int aif1_hw_params(struct snd_pcm_substream *substream, static int aif1_hw_free(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_pll(codec_dai, 0, @@ -231,7 +231,7 @@ static int bytcht_da7213_probe(struct platform_device *pdev) int ret_val = 0; int i; - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; card = &bytcht_da7213_card; card->dev = &pdev->dev; diff --git a/sound/soc/intel/boards/bytcht_es8316.c b/sound/soc/intel/boards/bytcht_es8316.c index 0adc5a5e134a..ddcd070100ef 100644 --- a/sound/soc/intel/boards/bytcht_es8316.c +++ b/sound/soc/intel/boards/bytcht_es8316.c @@ -157,7 +157,7 @@ static struct snd_soc_jack_pin byt_cht_es8316_jack_pins[] = { static int byt_cht_es8316_init(struct snd_soc_pcm_runtime *runtime) { - struct snd_soc_component *codec = runtime->codec_dai->component; + struct snd_soc_component *codec = asoc_rtd_to_codec(runtime, 0)->component; struct snd_soc_card *card = runtime->card; struct byt_cht_es8316_private *priv = snd_soc_card_get_drvdata(card); const struct snd_soc_dapm_route *custom_map; @@ -212,7 +212,7 @@ static int byt_cht_es8316_init(struct snd_soc_pcm_runtime *runtime) if (ret) dev_err(card->dev, "unable to enable MCLK\n"); - ret = snd_soc_dai_set_sysclk(runtime->codec_dai, 0, 19200000, + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_codec(runtime, 0), 0, 19200000, SND_SOC_CLOCK_IN); if (ret < 0) { dev_err(card->dev, "can't set codec clock %d\n", ret); @@ -262,7 +262,7 @@ static int byt_cht_es8316_codec_fixup(struct snd_soc_pcm_runtime *rtd, * with explicit setting to I2S 2ch 24-bit. The word length is set with * dai_set_tdm_slot() since there is no other API exposed */ - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBS_CFS @@ -272,7 +272,7 @@ static int byt_cht_es8316_codec_fixup(struct snd_soc_pcm_runtime *rtd, return ret; } - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, bits); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), 0x3, 0x3, 2, bits); if (ret < 0) { dev_err(rtd->dev, "can't set I2S config, err %d\n", ret); return ret; diff --git a/sound/soc/intel/boards/bytcht_nocodec.c b/sound/soc/intel/boards/bytcht_nocodec.c index 479af808ef43..8c0dab1f4030 100644 --- a/sound/soc/intel/boards/bytcht_nocodec.c +++ b/sound/soc/intel/boards/bytcht_nocodec.c @@ -58,7 +58,7 @@ static int codec_fixup(struct snd_soc_pcm_runtime *rtd, * with explicit setting to I2S 2ch 24-bit. The word length is set with * dai_set_tdm_slot() since there is no other API exposed */ - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBS_CFS); @@ -68,7 +68,7 @@ static int codec_fixup(struct snd_soc_pcm_runtime *rtd, return ret; } - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, 24); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), 0x3, 0x3, 2, 24); if (ret < 0) { dev_err(rtd->dev, "can't set I2S config, err %d\n", ret); return ret; diff --git a/sound/soc/intel/boards/bytcr_rt5640.c b/sound/soc/intel/boards/bytcr_rt5640.c index 6bd9ae813be2..33fb8ea4e5cb 100644 --- a/sound/soc/intel/boards/bytcr_rt5640.c +++ b/sound/soc/intel/boards/bytcr_rt5640.c @@ -381,7 +381,7 @@ static int byt_rt5640_aif1_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); return byt_rt5640_prepare_and_enable_pll1(dai, params_rate(params)); } @@ -805,7 +805,7 @@ static int byt_rt5640_init(struct snd_soc_pcm_runtime *runtime) { struct snd_soc_card *card = runtime->card; struct byt_rt5640_private *priv = snd_soc_card_get_drvdata(card); - struct snd_soc_component *component = runtime->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(runtime, 0)->component; const struct snd_soc_dapm_route *custom_map; int num_routes; int ret; @@ -962,7 +962,7 @@ static int byt_rt5640_codec_fixup(struct snd_soc_pcm_runtime *rtd, * with explicit setting to I2S 2ch. The word length is set with * dai_set_tdm_slot() since there is no other API exposed */ - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBS_CFS); @@ -971,7 +971,7 @@ static int byt_rt5640_codec_fixup(struct snd_soc_pcm_runtime *rtd, return ret; } - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, bits); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), 0x3, 0x3, 2, bits); if (ret < 0) { dev_err(rtd->dev, "can't set I2S config, err %d\n", ret); return ret; diff --git a/sound/soc/intel/boards/bytcr_rt5651.c b/sound/soc/intel/boards/bytcr_rt5651.c index 5074bb53f98e..214ef41e23e6 100644 --- a/sound/soc/intel/boards/bytcr_rt5651.c +++ b/sound/soc/intel/boards/bytcr_rt5651.c @@ -348,7 +348,7 @@ static int byt_rt5651_aif1_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); snd_pcm_format_t format = params_format(params); int rate = params_rate(params); int bclk_ratio; @@ -540,7 +540,7 @@ static int byt_rt5651_add_codec_device_props(struct device *i2c_dev) static int byt_rt5651_init(struct snd_soc_pcm_runtime *runtime) { struct snd_soc_card *card = runtime->card; - struct snd_soc_component *codec = runtime->codec_dai->component; + struct snd_soc_component *codec = asoc_rtd_to_codec(runtime, 0)->component; struct byt_rt5651_private *priv = snd_soc_card_get_drvdata(card); const struct snd_soc_dapm_route *custom_map; int num_routes; @@ -685,7 +685,7 @@ static int byt_rt5651_codec_fixup(struct snd_soc_pcm_runtime *rtd, * with explicit setting to I2S 2ch. The word length is set with * dai_set_tdm_slot() since there is no other API exposed */ - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBS_CFS @@ -696,7 +696,7 @@ static int byt_rt5651_codec_fixup(struct snd_soc_pcm_runtime *rtd, return ret; } - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, bits); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), 0x3, 0x3, 2, bits); if (ret < 0) { dev_err(rtd->dev, "can't set I2S config, err %d\n", ret); return ret; diff --git a/sound/soc/intel/boards/cht_bsw_max98090_ti.c b/sound/soc/intel/boards/cht_bsw_max98090_ti.c index 70bb86f3342f..135701738a44 100644 --- a/sound/soc/intel/boards/cht_bsw_max98090_ti.c +++ b/sound/soc/intel/boards/cht_bsw_max98090_ti.c @@ -113,7 +113,7 @@ static int cht_aif1_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, M98090_REG_SYSTEM_CLOCK, @@ -257,7 +257,7 @@ static int cht_codec_fixup(struct snd_soc_pcm_runtime *rtd, int ret = 0; unsigned int fmt = 0; - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, 16); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), 0x3, 0x3, 2, 16); if (ret < 0) { dev_err(rtd->dev, "can't set cpu_dai slot fmt: %d\n", ret); return ret; @@ -266,7 +266,7 @@ static int cht_codec_fixup(struct snd_soc_pcm_runtime *rtd, fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBS_CFS; - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, fmt); + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), fmt); if (ret < 0) { dev_err(rtd->dev, "can't set cpu_dai set fmt: %d\n", ret); return ret; @@ -553,7 +553,7 @@ static int snd_cht_mc_probe(struct platform_device *pdev) /* override plaform name, if required */ snd_soc_card_cht.dev = &pdev->dev; - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; platform_name = mach->mach_params.platform; ret_val = snd_soc_fixup_dai_links_platform_name(&snd_soc_card_cht, diff --git a/sound/soc/intel/boards/cht_bsw_nau8824.c b/sound/soc/intel/boards/cht_bsw_nau8824.c index 501bad3976fb..f456150f89c2 100644 --- a/sound/soc/intel/boards/cht_bsw_nau8824.c +++ b/sound/soc/intel/boards/cht_bsw_nau8824.c @@ -73,7 +73,7 @@ static int cht_aif1_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, NAU8824_CLK_FLL_FS, 0, @@ -96,7 +96,7 @@ static int cht_codec_init(struct snd_soc_pcm_runtime *runtime) { struct cht_mc_private *ctx = snd_soc_card_get_drvdata(runtime->card); struct snd_soc_jack *jack = &ctx->jack; - struct snd_soc_dai *codec_dai = runtime->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(runtime, 0); struct snd_soc_component *component = codec_dai->component; int ret, jack_type; @@ -259,7 +259,7 @@ static int snd_cht_mc_probe(struct platform_device *pdev) /* override plaform name, if required */ snd_soc_card_cht.dev = &pdev->dev; - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; platform_name = mach->mach_params.platform; ret_val = snd_soc_fixup_dai_links_platform_name(&snd_soc_card_cht, diff --git a/sound/soc/intel/boards/cht_bsw_rt5645.c b/sound/soc/intel/boards/cht_bsw_rt5645.c index b5b016d493f1..e64eca56e426 100644 --- a/sound/soc/intel/boards/cht_bsw_rt5645.c +++ b/sound/soc/intel/boards/cht_bsw_rt5645.c @@ -208,7 +208,7 @@ static int cht_aif1_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; /* set codec PLL source to the 19.2MHz platform clock (MCLK) */ @@ -252,7 +252,7 @@ static int cht_codec_init(struct snd_soc_pcm_runtime *runtime) { struct snd_soc_card *card = runtime->card; struct cht_mc_private *ctx = snd_soc_card_get_drvdata(runtime->card); - struct snd_soc_component *component = runtime->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(runtime, 0)->component; int jack_type; int ret; @@ -359,7 +359,7 @@ static int cht_codec_fixup(struct snd_soc_pcm_runtime *rtd, * with explicit setting to I2S 2ch 16-bit. The word length is set with * dai_set_tdm_slot() since there is no other API exposed */ - ret = snd_soc_dai_set_fmt(rtd->cpu_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_cpu(rtd, 0), SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBS_CFS @@ -369,7 +369,7 @@ static int cht_codec_fixup(struct snd_soc_pcm_runtime *rtd, return ret; } - ret = snd_soc_dai_set_fmt(rtd->codec_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_codec(rtd, 0), SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBS_CFS @@ -379,7 +379,7 @@ static int cht_codec_fixup(struct snd_soc_pcm_runtime *rtd, return ret; } - ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, 0x3, 0x3, 2, 16); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), 0x3, 0x3, 2, 16); if (ret < 0) { dev_err(rtd->dev, "can't set I2S config, err %d\n", ret); return ret; @@ -393,7 +393,7 @@ static int cht_codec_fixup(struct snd_soc_pcm_runtime *rtd, /* * Default mode for SSP configuration is TDM 4 slot */ - ret = snd_soc_dai_set_fmt(rtd->codec_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_codec(rtd, 0), SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_IB_NF | SND_SOC_DAIFMT_CBS_CFS); @@ -403,7 +403,7 @@ static int cht_codec_fixup(struct snd_soc_pcm_runtime *rtd, } /* TDM 4 slots 24 bit, set Rx & Tx bitmask to 4 active slots */ - ret = snd_soc_dai_set_tdm_slot(rtd->codec_dai, 0xF, 0xF, 4, 24); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_codec(rtd, 0), 0xF, 0xF, 4, 24); if (ret < 0) { dev_err(rtd->dev, "can't set codec TDM slot %d\n", ret); return ret; @@ -539,7 +539,7 @@ static int snd_cht_mc_probe(struct platform_device *pdev) if (!drv) return -ENOMEM; - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; for (i = 0; i < ARRAY_SIZE(snd_soc_cards); i++) { if (acpi_dev_found(snd_soc_cards[i].codec_id) && diff --git a/sound/soc/intel/boards/cht_bsw_rt5672.c b/sound/soc/intel/boards/cht_bsw_rt5672.c index 9d657421730a..097023a3ec14 100644 --- a/sound/soc/intel/boards/cht_bsw_rt5672.c +++ b/sound/soc/intel/boards/cht_bsw_rt5672.c @@ -144,7 +144,7 @@ static int cht_aif1_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; /* set codec PLL source to the 19.2MHz platform clock (MCLK) */ @@ -176,7 +176,7 @@ static const struct acpi_gpio_mapping cht_rt5672_gpios[] = { static int cht_codec_init(struct snd_soc_pcm_runtime *runtime) { int ret; - struct snd_soc_dai *codec_dai = runtime->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(runtime, 0); struct snd_soc_component *component = codec_dai->component; struct cht_mc_private *ctx = snd_soc_card_get_drvdata(runtime->card); @@ -255,7 +255,7 @@ static int cht_codec_fixup(struct snd_soc_pcm_runtime *rtd, /* * Default mode for SSP configuration is TDM 4 slot */ - ret = snd_soc_dai_set_fmt(rtd->codec_dai, + ret = snd_soc_dai_set_fmt(asoc_rtd_to_codec(rtd, 0), SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_IB_NF | SND_SOC_DAIFMT_CBS_CFS); @@ -265,7 +265,7 @@ static int cht_codec_fixup(struct snd_soc_pcm_runtime *rtd, } /* TDM 4 slots 24 bit, set Rx & Tx bitmask to 4 active slots */ - ret = snd_soc_dai_set_tdm_slot(rtd->codec_dai, 0xF, 0xF, 4, 24); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_codec(rtd, 0), 0xF, 0xF, 4, 24); if (ret < 0) { dev_err(rtd->dev, "can't set codec TDM slot %d\n", ret); return ret; diff --git a/sound/soc/intel/boards/cml_rt1011_rt5682.c b/sound/soc/intel/boards/cml_rt1011_rt5682.c index dd80d0186a6c..8167b2977e1d 100644 --- a/sound/soc/intel/boards/cml_rt1011_rt5682.c +++ b/sound/soc/intel/boards/cml_rt1011_rt5682.c @@ -85,7 +85,7 @@ static const struct snd_soc_dapm_route cml_rt1011_rt5682_map[] = { static int cml_rt5682_codec_init(struct snd_soc_pcm_runtime *rtd) { struct card_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct snd_soc_jack *jack; int ret; @@ -125,7 +125,7 @@ static int cml_rt5682_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int clk_id, clk_freq, pll_out, ret; clk_id = RT5682_PLL1_S_MCLK; @@ -164,8 +164,7 @@ static int cml_rt1011_hw_params(struct snd_pcm_substream *substream, srate = params_rate(params); - for (i = 0; i < rtd->num_codecs; i++) { - codec_dai = rtd->codec_dais[i]; + for_each_rtd_codec_dais(rtd, i, codec_dai) { /* 100 Fs to drive 24 bit data */ ret = snd_soc_dai_set_pll(codec_dai, 0, RT1011_PLL1_S_BCLK, @@ -275,7 +274,7 @@ static int sof_card_late_probe(struct snd_soc_card *card) static int hdmi_init(struct snd_soc_pcm_runtime *rtd) { struct card_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -447,12 +446,12 @@ static int snd_cml_rt1011_probe(struct platform_device *pdev) const char *platform_name; int ret; - ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_ATOMIC); + ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; INIT_LIST_HEAD(&ctx->hdmi_pcm_list); - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; snd_soc_card_cml.dev = &pdev->dev; platform_name = mach->mach_params.platform; diff --git a/sound/soc/intel/boards/glk_rt5682_max98357a.c b/sound/soc/intel/boards/glk_rt5682_max98357a.c index 8e947bad143c..f13158e4a1fc 100644 --- a/sound/soc/intel/boards/glk_rt5682_max98357a.c +++ b/sound/soc/intel/boards/glk_rt5682_max98357a.c @@ -136,8 +136,8 @@ static int geminilake_ssp_fixup(struct snd_soc_pcm_runtime *rtd, static int geminilake_rt5682_codec_init(struct snd_soc_pcm_runtime *rtd) { struct glk_card_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_jack *jack; int ret; @@ -188,7 +188,7 @@ static int geminilake_rt5682_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; /* Set valid bitmask & configuration for I2S in 24 bit */ @@ -208,7 +208,7 @@ static struct snd_soc_ops geminilake_rt5682_ops = { static int geminilake_hdmi_init(struct snd_soc_pcm_runtime *rtd) { struct glk_card_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct glk_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -225,7 +225,7 @@ static int geminilake_hdmi_init(struct snd_soc_pcm_runtime *rtd) static int geminilake_rt5682_fe_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; struct snd_soc_dapm_context *dapm; int ret; @@ -409,6 +409,7 @@ static struct snd_soc_dai_link geminilake_dais[] = { .init = NULL, .capture_only = 1, .nonatomic = 1, + .dynamic = 1, SND_SOC_DAILINK_REG(echoref, dummy, platform), }, [GLK_DPCM_AUDIO_REF_CP] = { @@ -604,7 +605,7 @@ static int geminilake_audio_probe(struct platform_device *pdev) snd_soc_card_set_drvdata(card, ctx); /* override plaform name, if required */ - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; platform_name = mach->mach_params.platform; ret = snd_soc_fixup_dai_links_platform_name(card, platform_name); diff --git a/sound/soc/intel/boards/haswell.c b/sound/soc/intel/boards/haswell.c index 3dadf9bff796..3ed53d7db4e6 100644 --- a/sound/soc/intel/boards/haswell.c +++ b/sound/soc/intel/boards/haswell.c @@ -56,7 +56,7 @@ static int haswell_rt5640_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, RT5640_SCLK_S_MCLK, 12288000, @@ -193,7 +193,7 @@ static int haswell_audio_probe(struct platform_device *pdev) haswell_rt5640.dev = &pdev->dev; /* override plaform name, if required */ - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; ret = snd_soc_fixup_dai_links_platform_name(&haswell_rt5640, mach->mach_params.platform); if (ret) diff --git a/sound/soc/intel/boards/kbl_da7219_max98357a.c b/sound/soc/intel/boards/kbl_da7219_max98357a.c index bc7f9a9ce9af..32cd90b8d4c4 100644 --- a/sound/soc/intel/boards/kbl_da7219_max98357a.c +++ b/sound/soc/intel/boards/kbl_da7219_max98357a.c @@ -159,8 +159,8 @@ static int kabylake_ssp_fixup(struct snd_soc_pcm_runtime *rtd, static int kabylake_da7219_codec_init(struct snd_soc_pcm_runtime *rtd) { struct kbl_codec_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_jack *jack; int ret; @@ -203,7 +203,7 @@ static int kabylake_da7219_codec_init(struct snd_soc_pcm_runtime *rtd) static int kabylake_hdmi_init(struct snd_soc_pcm_runtime *rtd, int device) { struct kbl_codec_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct kbl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -236,7 +236,7 @@ static int kabylake_hdmi3_init(struct snd_soc_pcm_runtime *rtd) static int kabylake_da7219_fe_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_dapm_context *dapm; - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; dapm = snd_soc_component_get_dapm(component); snd_soc_dapm_ignore_suspend(dapm, "Reference Capture"); diff --git a/sound/soc/intel/boards/kbl_da7219_max98927.c b/sound/soc/intel/boards/kbl_da7219_max98927.c index 7a13e9b35187..abd4e3839678 100644 --- a/sound/soc/intel/boards/kbl_da7219_max98927.c +++ b/sound/soc/intel/boards/kbl_da7219_max98927.c @@ -176,10 +176,10 @@ static int kabylake_ssp0_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *runtime = substream->private_data; + struct snd_soc_dai *codec_dai; int ret, j; - for (j = 0; j < runtime->num_codecs; j++) { - struct snd_soc_dai *codec_dai = runtime->codec_dais[j]; + for_each_rtd_codec_dais(runtime, j, codec_dai) { if (!strcmp(codec_dai->component->name, MAX98927_DEV0_NAME)) { ret = snd_soc_dai_set_tdm_slot(codec_dai, 0x30, 3, 8, 16); @@ -221,10 +221,10 @@ static int kabylake_ssp0_hw_params(struct snd_pcm_substream *substream, static int kabylake_ssp0_trigger(struct snd_pcm_substream *substream, int cmd) { struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *codec_dai; int j, ret; - for (j = 0; j < rtd->num_codecs; j++) { - struct snd_soc_dai *codec_dai = rtd->codec_dais[j]; + for_each_rtd_codec_dais(rtd, j, codec_dai) { const char *name = codec_dai->component->name; struct snd_soc_component *component = codec_dai->component; struct snd_soc_dapm_context *dapm = @@ -331,7 +331,7 @@ static int kabylake_ssp_fixup(struct snd_soc_pcm_runtime *rtd, static int kabylake_da7219_codec_init(struct snd_soc_pcm_runtime *rtd) { struct kbl_codec_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct snd_soc_jack *jack; struct snd_soc_card *card = rtd->card; int ret; @@ -381,7 +381,7 @@ static int kabylake_dmic_init(struct snd_soc_pcm_runtime *rtd) static int kabylake_hdmi_init(struct snd_soc_pcm_runtime *rtd, int device) { struct kbl_codec_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct kbl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -414,7 +414,7 @@ static int kabylake_hdmi3_init(struct snd_soc_pcm_runtime *rtd) static int kabylake_da7219_fe_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_dapm_context *dapm; - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; dapm = snd_soc_component_get_dapm(component); snd_soc_dapm_ignore_suspend(dapm, "Reference Capture"); diff --git a/sound/soc/intel/boards/kbl_rt5660.c b/sound/soc/intel/boards/kbl_rt5660.c index e23dea9ab79a..6460e3f0c974 100644 --- a/sound/soc/intel/boards/kbl_rt5660.c +++ b/sound/soc/intel/boards/kbl_rt5660.c @@ -157,7 +157,7 @@ static int kabylake_rt5660_codec_init(struct snd_soc_pcm_runtime *rtd) { int ret; struct kbl_codec_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component); ret = devm_acpi_dev_add_driver_gpios(component->dev, acpi_rt5660_gpios); @@ -210,7 +210,7 @@ static int kabylake_rt5660_codec_init(struct snd_soc_pcm_runtime *rtd) static int kabylake_hdmi_init(struct snd_soc_pcm_runtime *rtd, int device) { struct kbl_codec_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct kbl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -244,7 +244,7 @@ static int kabylake_rt5660_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, diff --git a/sound/soc/intel/boards/kbl_rt5663_max98927.c b/sound/soc/intel/boards/kbl_rt5663_max98927.c index d8f2ff7139a9..658a9da3a40f 100644 --- a/sound/soc/intel/boards/kbl_rt5663_max98927.c +++ b/sound/soc/intel/boards/kbl_rt5663_max98927.c @@ -242,7 +242,7 @@ static int kabylake_rt5663_fe_init(struct snd_soc_pcm_runtime *rtd) { int ret; struct snd_soc_dapm_context *dapm; - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; dapm = snd_soc_component_get_dapm(component); ret = snd_soc_dapm_ignore_suspend(dapm, "Reference Capture"); @@ -258,7 +258,7 @@ static int kabylake_rt5663_codec_init(struct snd_soc_pcm_runtime *rtd) { int ret; struct kbl_rt5663_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct snd_soc_jack *jack; /* @@ -305,7 +305,7 @@ static int kabylake_rt5663_max98927_codec_init(struct snd_soc_pcm_runtime *rtd) static int kabylake_hdmi_init(struct snd_soc_pcm_runtime *rtd, int device) { struct kbl_rt5663_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct kbl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -431,7 +431,7 @@ static int kabylake_rt5663_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; /* use ASRC for internal clocks, as PLL rate isn't multiple of BCLK */ @@ -472,7 +472,7 @@ static int kabylake_ssp0_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *codec_dai; int ret = 0, j; - for_each_rtd_codec_dai(rtd, j, codec_dai) { + for_each_rtd_codec_dais(rtd, j, codec_dai) { if (!strcmp(codec_dai->component->name, MAXIM_DEV0_NAME)) { /* * Use channel 4 and 5 for the first amp @@ -962,7 +962,7 @@ static int kabylake_audio_probe(struct platform_device *pdev) kabylake_audio_card->dev = &pdev->dev; snd_soc_card_set_drvdata(kabylake_audio_card, ctx); - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; if (mach) dmic_constraints = mach->mach_params.dmic_num == 2 ? &constraints_dmic_2ch : &constraints_dmic_channels; diff --git a/sound/soc/intel/boards/kbl_rt5663_rt5514_max98927.c b/sound/soc/intel/boards/kbl_rt5663_rt5514_max98927.c index 96c814f36458..1b1f8d7a4ea3 100644 --- a/sound/soc/intel/boards/kbl_rt5663_rt5514_max98927.c +++ b/sound/soc/intel/boards/kbl_rt5663_rt5514_max98927.c @@ -206,7 +206,7 @@ static struct snd_soc_codec_conf max98927_codec_conf[] = { static int kabylake_rt5663_fe_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_dapm_context *dapm; - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; int ret; dapm = snd_soc_component_get_dapm(component); @@ -221,7 +221,7 @@ static int kabylake_rt5663_codec_init(struct snd_soc_pcm_runtime *rtd) { int ret; struct kbl_codec_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct snd_soc_jack *jack; /* @@ -255,7 +255,7 @@ static int kabylake_rt5663_codec_init(struct snd_soc_pcm_runtime *rtd) static int kabylake_hdmi_init(struct snd_soc_pcm_runtime *rtd, int device) { struct kbl_codec_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct kbl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -372,7 +372,7 @@ static int kabylake_rt5663_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; /* use ASRC for internal clocks, as PLL rate isn't multiple of BCLK */ @@ -399,7 +399,7 @@ static int kabylake_ssp0_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *codec_dai; int ret = 0, j; - for_each_rtd_codec_dai(rtd, j, codec_dai) { + for_each_rtd_codec_dais(rtd, j, codec_dai) { if (!strcmp(codec_dai->component->name, RT5514_DEV_NAME)) { ret = snd_soc_dai_set_tdm_slot(codec_dai, 0xF, 0, 8, 16); if (ret < 0) { @@ -772,7 +772,7 @@ static int kabylake_audio_probe(struct platform_device *pdev) kabylake_audio_card.dev = &pdev->dev; snd_soc_card_set_drvdata(&kabylake_audio_card, ctx); - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; if (mach) dmic_constraints = mach->mach_params.dmic_num == 2 ? &constraints_dmic_2ch : &constraints_dmic_channels; diff --git a/sound/soc/intel/boards/skl_hda_dsp_common.h b/sound/soc/intel/boards/skl_hda_dsp_common.h index d6150670ca05..e8545d13062f 100644 --- a/sound/soc/intel/boards/skl_hda_dsp_common.h +++ b/sound/soc/intel/boards/skl_hda_dsp_common.h @@ -49,6 +49,10 @@ static inline int skl_hda_hdmi_build_controls(struct snd_soc_card *card) struct snd_soc_component *component; struct skl_hda_hdmi_pcm *pcm; + /* HDMI disabled, do not create controls */ + if (list_empty(&ctx->hdmi_pcm_list)) + return 0; + pcm = list_first_entry(&ctx->hdmi_pcm_list, struct skl_hda_hdmi_pcm, head); component = pcm->codec_dai->component; diff --git a/sound/soc/intel/boards/skl_hda_dsp_generic.c b/sound/soc/intel/boards/skl_hda_dsp_generic.c index 11eaee9ae41f..3be764299ab0 100644 --- a/sound/soc/intel/boards/skl_hda_dsp_generic.c +++ b/sound/soc/intel/boards/skl_hda_dsp_generic.c @@ -61,6 +61,9 @@ static const struct snd_soc_dapm_route skl_hda_map[] = { { "Alt Analog CPU Capture", NULL, "Alt Analog Codec Capture" }, }; +SND_SOC_DAILINK_DEF(dummy_codec, + DAILINK_COMP_ARRAY(COMP_CODEC("snd-soc-dummy", "snd-soc-dummy-dai"))); + static int skl_hda_card_late_probe(struct snd_soc_card *card) { return skl_hda_hdmi_jack_init(card); @@ -114,13 +117,19 @@ static int skl_hda_fill_card_info(struct snd_soc_acpi_mach_params *mach_params) { struct snd_soc_card *card = &hda_soc_card; struct snd_soc_dai_link *dai_link; - u32 codec_count, codec_mask; + u32 codec_count, codec_mask, idisp_mask; int i, num_links, num_route; codec_mask = mach_params->codec_mask; codec_count = hweight_long(codec_mask); + idisp_mask = codec_mask & IDISP_CODEC_MASK; - if (codec_count == 1 && codec_mask & IDISP_CODEC_MASK) { + if (!codec_count || codec_count > 2 || + (codec_count == 2 && !idisp_mask)) + return -EINVAL; + + if (codec_mask == idisp_mask) { + /* topology with iDisp as the only HDA codec */ num_links = IDISP_DAI_COUNT + DMIC_DAI_COUNT; num_route = IDISP_ROUTE_COUNT; @@ -135,13 +144,19 @@ static int skl_hda_fill_card_info(struct snd_soc_acpi_mach_params *mach_params) skl_hda_be_dai_links[IDISP_DAI_COUNT + HDAC_DAI_COUNT + i]; } - } else if (codec_count == 2 && codec_mask & IDISP_CODEC_MASK) { + } else { + /* topology with external and iDisp HDA codecs */ num_links = ARRAY_SIZE(skl_hda_be_dai_links); num_route = ARRAY_SIZE(skl_hda_map); card->dapm_widgets = skl_hda_widgets; card->num_dapm_widgets = ARRAY_SIZE(skl_hda_widgets); - } else { - return -EINVAL; + if (!idisp_mask) { + for (i = 0; i < IDISP_DAI_COUNT; i++) { + skl_hda_be_dai_links[i].codecs = dummy_codec; + skl_hda_be_dai_links[i].num_codecs = + ARRAY_SIZE(dummy_codec); + } + } } card->num_links = num_links; @@ -167,7 +182,7 @@ static int skl_hda_audio_probe(struct platform_device *pdev) INIT_LIST_HEAD(&ctx->hdmi_pcm_list); - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; if (!mach) return -EINVAL; diff --git a/sound/soc/intel/boards/skl_nau88l25_max98357a.c b/sound/soc/intel/boards/skl_nau88l25_max98357a.c index e6de3b28d840..d7b8154c43a4 100644 --- a/sound/soc/intel/boards/skl_nau88l25_max98357a.c +++ b/sound/soc/intel/boards/skl_nau88l25_max98357a.c @@ -157,7 +157,7 @@ static int skylake_ssp_fixup(struct snd_soc_pcm_runtime *rtd, static int skylake_nau8825_codec_init(struct snd_soc_pcm_runtime *rtd) { int ret; - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; /* * Headset buttons map to the google Reference headset. @@ -182,7 +182,7 @@ static int skylake_nau8825_codec_init(struct snd_soc_pcm_runtime *rtd) static int skylake_hdmi1_init(struct snd_soc_pcm_runtime *rtd) { struct skl_nau8825_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct skl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -200,7 +200,7 @@ static int skylake_hdmi1_init(struct snd_soc_pcm_runtime *rtd) static int skylake_hdmi2_init(struct snd_soc_pcm_runtime *rtd) { struct skl_nau8825_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct skl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -218,7 +218,7 @@ static int skylake_hdmi2_init(struct snd_soc_pcm_runtime *rtd) static int skylake_hdmi3_init(struct snd_soc_pcm_runtime *rtd) { struct skl_nau8825_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct skl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -236,7 +236,7 @@ static int skylake_hdmi3_init(struct snd_soc_pcm_runtime *rtd) static int skylake_nau8825_fe_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_dapm_context *dapm; - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; dapm = snd_soc_component_get_dapm(component); snd_soc_dapm_ignore_suspend(dapm, "Reference Capture"); @@ -296,7 +296,7 @@ static int skylake_nau8825_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, @@ -660,7 +660,7 @@ static int skylake_audio_probe(struct platform_device *pdev) skylake_audio_card.dev = &pdev->dev; snd_soc_card_set_drvdata(&skylake_audio_card, ctx); - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; if (mach) dmic_constraints = mach->mach_params.dmic_num == 2 ? &constraints_dmic_2ch : &constraints_dmic_channels; diff --git a/sound/soc/intel/boards/skl_nau88l25_ssm4567.c b/sound/soc/intel/boards/skl_nau88l25_ssm4567.c index c99c8b23e509..4b317bcf6ea0 100644 --- a/sound/soc/intel/boards/skl_nau88l25_ssm4567.c +++ b/sound/soc/intel/boards/skl_nau88l25_ssm4567.c @@ -161,12 +161,12 @@ static int skylake_ssm4567_codec_init(struct snd_soc_pcm_runtime *rtd) int ret; /* Slot 1 for left */ - ret = snd_soc_dai_set_tdm_slot(rtd->codec_dais[0], 0x01, 0x01, 2, 48); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_codec(rtd, 0), 0x01, 0x01, 2, 48); if (ret < 0) return ret; /* Slot 2 for right */ - ret = snd_soc_dai_set_tdm_slot(rtd->codec_dais[1], 0x02, 0x02, 2, 48); + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_codec(rtd, 1), 0x02, 0x02, 2, 48); if (ret < 0) return ret; @@ -176,7 +176,7 @@ static int skylake_ssm4567_codec_init(struct snd_soc_pcm_runtime *rtd) static int skylake_nau8825_codec_init(struct snd_soc_pcm_runtime *rtd) { int ret; - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; /* * 4 buttons here map to the google Reference headset @@ -201,7 +201,7 @@ static int skylake_nau8825_codec_init(struct snd_soc_pcm_runtime *rtd) static int skylake_hdmi1_init(struct snd_soc_pcm_runtime *rtd) { struct skl_nau88125_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct skl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -219,7 +219,7 @@ static int skylake_hdmi1_init(struct snd_soc_pcm_runtime *rtd) static int skylake_hdmi2_init(struct snd_soc_pcm_runtime *rtd) { struct skl_nau88125_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct skl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -238,7 +238,7 @@ static int skylake_hdmi2_init(struct snd_soc_pcm_runtime *rtd) static int skylake_hdmi3_init(struct snd_soc_pcm_runtime *rtd) { struct skl_nau88125_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct skl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -256,7 +256,7 @@ static int skylake_hdmi3_init(struct snd_soc_pcm_runtime *rtd) static int skylake_nau8825_fe_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_dapm_context *dapm; - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; dapm = snd_soc_component_get_dapm(component); snd_soc_dapm_ignore_suspend(dapm, "Reference Capture"); @@ -348,7 +348,7 @@ static int skylake_nau8825_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, @@ -686,6 +686,7 @@ static struct snd_soc_card skylake_audio_card = { .codec_conf = ssm4567_codec_conf, .num_configs = ARRAY_SIZE(ssm4567_codec_conf), .fully_routed = true, + .disable_route_checks = true, .late_probe = skylake_card_late_probe, }; @@ -703,7 +704,7 @@ static int skylake_audio_probe(struct platform_device *pdev) skylake_audio_card.dev = &pdev->dev; snd_soc_card_set_drvdata(&skylake_audio_card, ctx); - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; if (mach) dmic_constraints = mach->mach_params.dmic_num == 2 ? &constraints_dmic_2ch : &constraints_dmic_channels; diff --git a/sound/soc/intel/boards/skl_rt286.c b/sound/soc/intel/boards/skl_rt286.c index a9aec66a2351..903ae1b28ec9 100644 --- a/sound/soc/intel/boards/skl_rt286.c +++ b/sound/soc/intel/boards/skl_rt286.c @@ -112,7 +112,7 @@ static const struct snd_soc_dapm_route skylake_rt286_map[] = { static int skylake_rt286_fe_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_dapm_context *dapm; - struct snd_soc_component *component = rtd->cpu_dai->component; + struct snd_soc_component *component = asoc_rtd_to_cpu(rtd, 0)->component; dapm = snd_soc_component_get_dapm(component); snd_soc_dapm_ignore_suspend(dapm, "Reference Capture"); @@ -122,7 +122,7 @@ static int skylake_rt286_fe_init(struct snd_soc_pcm_runtime *rtd) static int skylake_rt286_codec_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; int ret; ret = snd_soc_card_jack_new(rtd->card, "Headset", @@ -143,7 +143,7 @@ static int skylake_rt286_codec_init(struct snd_soc_pcm_runtime *rtd) static int skylake_hdmi_init(struct snd_soc_pcm_runtime *rtd) { struct skl_rt286_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct skl_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -229,7 +229,7 @@ static int skylake_rt286_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, RT286_SCLK_S_PLL, 24000000, diff --git a/sound/soc/intel/boards/sof_da7219_max98373.c b/sound/soc/intel/boards/sof_da7219_max98373.c index 8f44f13d2848..b707dd3b5625 100644 --- a/sound/soc/intel/boards/sof_da7219_max98373.c +++ b/sound/soc/intel/boards/sof_da7219_max98373.c @@ -2,7 +2,7 @@ // Copyright(c) 2019 Intel Corporation. /* - * Intel SOF Machine driver for DA7219 + MAX98373 codec + * Intel SOF Machine driver for DA7219 + MAX98373/MAX98360A codec */ #include @@ -69,11 +69,20 @@ static const struct snd_kcontrol_new controls[] = { SOC_DAPM_PIN_SWITCH("Right Spk"), }; +static const struct snd_kcontrol_new m98360a_controls[] = { + SOC_DAPM_PIN_SWITCH("Headphone Jack"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), + SOC_DAPM_PIN_SWITCH("Spk"), +}; + +/* For MAX98373 amp */ static const struct snd_soc_dapm_widget widgets[] = { SND_SOC_DAPM_HP("Headphone Jack", NULL), SND_SOC_DAPM_MIC("Headset Mic", NULL), + SND_SOC_DAPM_SPK("Left Spk", NULL), SND_SOC_DAPM_SPK("Right Spk", NULL), + SND_SOC_DAPM_SUPPLY("Platform Clock", SND_SOC_NOPM, 0, 0, platform_clock_control, SND_SOC_DAPM_POST_PMD | SND_SOC_DAPM_PRE_PMU), @@ -83,21 +92,45 @@ static const struct snd_soc_dapm_route audio_map[] = { { "Headphone Jack", NULL, "HPL" }, { "Headphone Jack", NULL, "HPR" }, + { "MIC", NULL, "Headset Mic" }, + + { "Headphone Jack", NULL, "Platform Clock" }, + { "Headset Mic", NULL, "Platform Clock" }, + { "Left Spk", NULL, "Left BE_OUT" }, { "Right Spk", NULL, "Right BE_OUT" }, +}; + +/* For MAX98360A amp */ +static const struct snd_soc_dapm_widget max98360a_widgets[] = { + SND_SOC_DAPM_HP("Headphone Jack", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), + + SND_SOC_DAPM_SPK("Spk", NULL), + + SND_SOC_DAPM_SUPPLY("Platform Clock", SND_SOC_NOPM, 0, 0, + platform_clock_control, SND_SOC_DAPM_POST_PMD | + SND_SOC_DAPM_PRE_PMU), +}; + +static const struct snd_soc_dapm_route max98360a_map[] = { + { "Headphone Jack", NULL, "HPL" }, + { "Headphone Jack", NULL, "HPR" }, { "MIC", NULL, "Headset Mic" }, { "Headphone Jack", NULL, "Platform Clock" }, { "Headset Mic", NULL, "Platform Clock" }, + + {"Spk", NULL, "Speaker"}, }; static struct snd_soc_jack headset; static int da7219_codec_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_component *component = codec_dai->component; struct snd_soc_jack *jack; int ret; @@ -140,7 +173,7 @@ static int ssp1_hw_params(struct snd_pcm_substream *substream, int ret, j; for (j = 0; j < runtime->num_codecs; j++) { - struct snd_soc_dai *codec_dai = runtime->codec_dais[j]; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(runtime, j); if (!strcmp(codec_dai->component->name, MAXIM_DEV0_NAME)) { /* vmon_slot_no = 0 imon_slot_no = 1 for TX slots */ @@ -181,7 +214,7 @@ static struct snd_soc_codec_conf max98373_codec_conf[] = { static int hdmi_init(struct snd_soc_pcm_runtime *rtd) { struct card_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -224,6 +257,9 @@ SND_SOC_DAILINK_DEF(ssp1_amps, /* Left */ COMP_CODEC(MAXIM_DEV0_NAME, MAX98373_CODEC_DAI), /* Right */ COMP_CODEC(MAXIM_DEV1_NAME, MAX98373_CODEC_DAI))); +SND_SOC_DAILINK_DEF(ssp1_m98360a, + DAILINK_COMP_ARRAY(COMP_CODEC("MX98360A:00", "HiFi"))); + SND_SOC_DAILINK_DEF(dmic_pin, DAILINK_COMP_ARRAY(COMP_CPU("DMIC01 Pin"))); SND_SOC_DAILINK_DEF(dmic_codec, @@ -320,6 +356,21 @@ static struct snd_soc_card card_da7219_m98373 = { .late_probe = card_late_probe, }; +static struct snd_soc_card card_da7219_m98360a = { + .name = "da7219max98360a", + .owner = THIS_MODULE, + .dai_link = dais, + .num_links = ARRAY_SIZE(dais), + .controls = m98360a_controls, + .num_controls = ARRAY_SIZE(m98360a_controls), + .dapm_widgets = max98360a_widgets, + .num_dapm_widgets = ARRAY_SIZE(max98360a_widgets), + .dapm_routes = max98360a_map, + .num_dapm_routes = ARRAY_SIZE(max98360a_map), + .fully_routed = true, + .late_probe = card_late_probe, +}; + static int audio_probe(struct platform_device *pdev) { static struct snd_soc_card *card; @@ -327,15 +378,26 @@ static int audio_probe(struct platform_device *pdev) struct card_private *ctx; int ret; - ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_ATOMIC); + ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; + /* By default dais[0] is configured for max98373 */ + if (!strcmp(pdev->name, "sof_da7219_max98360a")) { + dais[0] = (struct snd_soc_dai_link) { + .name = "SSP1-Codec", + .id = 0, + .no_pcm = 1, + .dpcm_playback = 1, + .ignore_pmdown_time = 1, + SND_SOC_DAILINK_REG(ssp1_pin, ssp1_m98360a, platform) }; + } + INIT_LIST_HEAD(&ctx->hdmi_pcm_list); card = (struct snd_soc_card *)pdev->id_entry->driver_data; card->dev = &pdev->dev; - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; ret = snd_soc_fixup_dai_links_platform_name(card, mach->mach_params.platform); if (ret) @@ -351,13 +413,17 @@ static const struct platform_device_id board_ids[] = { .name = "sof_da7219_max98373", .driver_data = (kernel_ulong_t)&card_da7219_m98373, }, + { + .name = "sof_da7219_max98360a", + .driver_data = (kernel_ulong_t)&card_da7219_m98360a, + }, { } }; static struct platform_driver audio = { .probe = audio_probe, .driver = { - .name = "sof_da7219_max98373", + .name = "sof_da7219_max98_360a_373", .pm = &snd_soc_pm_ops, }, .id_table = board_ids, @@ -368,4 +434,5 @@ module_platform_driver(audio) MODULE_DESCRIPTION("ASoC Intel(R) SOF Machine driver"); MODULE_AUTHOR("Yong Zhi "); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:sof_da7219_max98360a"); MODULE_ALIAS("platform:sof_da7219_max98373"); diff --git a/sound/soc/intel/boards/sof_maxim_common.c b/sound/soc/intel/boards/sof_maxim_common.c new file mode 100644 index 000000000000..463b39a7ccfd --- /dev/null +++ b/sound/soc/intel/boards/sof_maxim_common.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright(c) 2020 Intel Corporation. All rights reserved. +#include +#include +#include +#include +#include +#include +#include "sof_maxim_common.h" + +static const struct snd_soc_dapm_route max_98373_dapm_routes[] = { + /* speaker */ + { "Left Spk", NULL, "Left BE_OUT" }, + { "Right Spk", NULL, "Right BE_OUT" }, +}; + +static struct snd_soc_codec_conf max_98373_codec_conf[] = { + { + .dlc = COMP_CODEC_CONF(MAX_98373_DEV0_NAME), + .name_prefix = "Right", + }, + { + .dlc = COMP_CODEC_CONF(MAX_98373_DEV1_NAME), + .name_prefix = "Left", + }, +}; + +struct snd_soc_dai_link_component max_98373_components[] = { + { /* For Left */ + .name = MAX_98373_DEV0_NAME, + .dai_name = MAX_98373_CODEC_DAI, + }, + { /* For Right */ + .name = MAX_98373_DEV1_NAME, + .dai_name = MAX_98373_CODEC_DAI, + }, +}; + +static int max98373_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *codec_dai; + int j; + + for_each_rtd_codec_dais(rtd, j, codec_dai) { + if (!strcmp(codec_dai->component->name, MAX_98373_DEV0_NAME)) { + /* DEV0 tdm slot configuration */ + snd_soc_dai_set_tdm_slot(codec_dai, 0x30, 3, 8, 16); + } + if (!strcmp(codec_dai->component->name, MAX_98373_DEV1_NAME)) { + /* DEV1 tdm slot configuration */ + snd_soc_dai_set_tdm_slot(codec_dai, 0xC0, 3, 8, 16); + } + } + return 0; +} + +struct snd_soc_ops max_98373_ops = { + .hw_params = max98373_hw_params, +}; + +int max98373_spk_codec_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + int ret; + + ret = snd_soc_dapm_add_routes(&card->dapm, max_98373_dapm_routes, + ARRAY_SIZE(max_98373_dapm_routes)); + if (ret) + dev_err(rtd->dev, "Speaker map addition failed: %d\n", ret); + return ret; +} + +void sof_max98373_codec_conf(struct snd_soc_card *card) +{ + card->codec_conf = max_98373_codec_conf; + card->num_configs = ARRAY_SIZE(max_98373_codec_conf); +} diff --git a/sound/soc/intel/boards/sof_maxim_common.h b/sound/soc/intel/boards/sof_maxim_common.h new file mode 100644 index 000000000000..406bf0e81155 --- /dev/null +++ b/sound/soc/intel/boards/sof_maxim_common.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(c) 2020 Intel Corporation. + */ + +/* + * This file defines data structures used in Machine Driver for Intel + * platforms with Maxim Codecs. + */ +#ifndef __SOF_MAXIM_COMMON_H +#define __SOF_MAXIM_COMMON_H + +#include + +#define MAX_98373_CODEC_DAI "max98373-aif1" +#define MAX_98373_DEV0_NAME "i2c-MX98373:00" +#define MAX_98373_DEV1_NAME "i2c-MX98373:01" + +extern struct snd_soc_dai_link_component max_98373_components[2]; +extern struct snd_soc_ops max_98373_ops; + +int max98373_spk_codec_init(struct snd_soc_pcm_runtime *rtd); +void sof_max98373_codec_conf(struct snd_soc_card *card); +#endif /* __SOF_MAXIM_COMMON_H */ diff --git a/sound/soc/intel/boards/sof_pcm512x.c b/sound/soc/intel/boards/sof_pcm512x.c new file mode 100644 index 000000000000..fb7811899999 --- /dev/null +++ b/sound/soc/intel/boards/sof_pcm512x.c @@ -0,0 +1,448 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2018-2020 Intel Corporation. + +/* + * Intel SOF Machine Driver for Intel platforms with TI PCM512x codec, + * e.g. Up or Up2 with Hifiberry DAC+ HAT + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../codecs/pcm512x.h" +#include "../common/soc-intel-quirks.h" +#include "hda_dsp_common.h" + +#define NAME_SIZE 32 + +#define SOF_PCM512X_SSP_CODEC(quirk) ((quirk) & GENMASK(3, 0)) +#define SOF_PCM512X_SSP_CODEC_MASK (GENMASK(3, 0)) + +#define IDISP_CODEC_MASK 0x4 + +/* Default: SSP5 */ +static unsigned long sof_pcm512x_quirk = SOF_PCM512X_SSP_CODEC(5); + +static bool is_legacy_cpu; + +struct sof_hdmi_pcm { + struct list_head head; + struct snd_soc_dai *codec_dai; + int device; +}; + +struct sof_card_private { + struct list_head hdmi_pcm_list; + bool idisp_codec; +}; + +static int sof_pcm512x_quirk_cb(const struct dmi_system_id *id) +{ + sof_pcm512x_quirk = (unsigned long)id->driver_data; + return 1; +} + +static const struct dmi_system_id sof_pcm512x_quirk_table[] = { + { + .callback = sof_pcm512x_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "AAEON"), + DMI_MATCH(DMI_PRODUCT_NAME, "UP-CHT01"), + }, + .driver_data = (void *)(SOF_PCM512X_SSP_CODEC(2)), + }, + {} +}; + +static int sof_hdmi_init(struct snd_soc_pcm_runtime *rtd) +{ + struct sof_card_private *ctx = snd_soc_card_get_drvdata(rtd->card); + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); + struct sof_hdmi_pcm *pcm; + + pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); + if (!pcm) + return -ENOMEM; + + /* dai_link id is 1:1 mapped to the PCM device */ + pcm->device = rtd->dai_link->id; + pcm->codec_dai = dai; + + list_add_tail(&pcm->head, &ctx->hdmi_pcm_list); + + return 0; +} + +static int sof_pcm512x_codec_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_component *codec = asoc_rtd_to_codec(rtd, 0)->component; + + snd_soc_component_update_bits(codec, PCM512x_GPIO_EN, 0x08, 0x08); + snd_soc_component_update_bits(codec, PCM512x_GPIO_OUTPUT_4, 0x0f, 0x02); + snd_soc_component_update_bits(codec, PCM512x_GPIO_CONTROL_1, + 0x08, 0x08); + + return 0; +} + +static int aif1_startup(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_component *codec = asoc_rtd_to_codec(rtd, 0)->component; + + snd_soc_component_update_bits(codec, PCM512x_GPIO_CONTROL_1, + 0x08, 0x08); + + return 0; +} + +static void aif1_shutdown(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_component *codec = asoc_rtd_to_codec(rtd, 0)->component; + + snd_soc_component_update_bits(codec, PCM512x_GPIO_CONTROL_1, + 0x08, 0x00); +} + +static const struct snd_soc_ops sof_pcm512x_ops = { + .startup = aif1_startup, + .shutdown = aif1_shutdown, +}; + +static struct snd_soc_dai_link_component platform_component[] = { + { + /* name might be overridden during probe */ + .name = "0000:00:1f.3" + } +}; + +#if IS_ENABLED(CONFIG_SND_HDA_CODEC_HDMI) +static int sof_card_late_probe(struct snd_soc_card *card) +{ + struct sof_card_private *ctx = snd_soc_card_get_drvdata(card); + struct sof_hdmi_pcm *pcm; + + /* HDMI is not supported by SOF on Baytrail/CherryTrail */ + if (is_legacy_cpu) + return 0; + + if (list_empty(&ctx->hdmi_pcm_list)) + return -EINVAL; + + if (!ctx->idisp_codec) + return 0; + + pcm = list_first_entry(&ctx->hdmi_pcm_list, struct sof_hdmi_pcm, head); + + return hda_dsp_hdmi_build_controls(card, pcm->codec_dai->component); +} +#else +static int sof_card_late_probe(struct snd_soc_card *card) +{ + return 0; +} +#endif + +static const struct snd_kcontrol_new sof_controls[] = { + SOC_DAPM_PIN_SWITCH("Ext Spk"), +}; + +static const struct snd_soc_dapm_widget sof_widgets[] = { + SND_SOC_DAPM_SPK("Ext Spk", NULL), +}; + +static const struct snd_soc_dapm_widget dmic_widgets[] = { + SND_SOC_DAPM_MIC("SoC DMIC", NULL), +}; + +static const struct snd_soc_dapm_route sof_map[] = { + /* Speaker */ + {"Ext Spk", NULL, "OUTR"}, + {"Ext Spk", NULL, "OUTL"}, +}; + +static const struct snd_soc_dapm_route dmic_map[] = { + /* digital mics */ + {"DMic", NULL, "SoC DMIC"}, +}; + +static int dmic_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + int ret; + + ret = snd_soc_dapm_new_controls(&card->dapm, dmic_widgets, + ARRAY_SIZE(dmic_widgets)); + if (ret) { + dev_err(card->dev, "DMic widget addition failed: %d\n", ret); + /* Don't need to add routes if widget addition failed */ + return ret; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, dmic_map, + ARRAY_SIZE(dmic_map)); + + if (ret) + dev_err(card->dev, "DMic map addition failed: %d\n", ret); + + return ret; +} + +/* sof audio machine driver for pcm512x codec */ +static struct snd_soc_card sof_audio_card_pcm512x = { + .name = "pcm512x", + .owner = THIS_MODULE, + .controls = sof_controls, + .num_controls = ARRAY_SIZE(sof_controls), + .dapm_widgets = sof_widgets, + .num_dapm_widgets = ARRAY_SIZE(sof_widgets), + .dapm_routes = sof_map, + .num_dapm_routes = ARRAY_SIZE(sof_map), + .fully_routed = true, + .late_probe = sof_card_late_probe, +}; + +SND_SOC_DAILINK_DEF(pcm512x_component, + DAILINK_COMP_ARRAY(COMP_CODEC("i2c-104C5122:00", "pcm512x-hifi"))); +SND_SOC_DAILINK_DEF(dmic_component, + DAILINK_COMP_ARRAY(COMP_CODEC("dmic-codec", "dmic-hifi"))); + +static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, + int ssp_codec, + int dmic_be_num, + int hdmi_num, + bool idisp_codec) +{ + struct snd_soc_dai_link_component *idisp_components; + struct snd_soc_dai_link_component *cpus; + struct snd_soc_dai_link *links; + int i, id = 0; + + links = devm_kcalloc(dev, sof_audio_card_pcm512x.num_links, + sizeof(struct snd_soc_dai_link), GFP_KERNEL); + cpus = devm_kcalloc(dev, sof_audio_card_pcm512x.num_links, + sizeof(struct snd_soc_dai_link_component), GFP_KERNEL); + if (!links || !cpus) + goto devm_err; + + /* codec SSP */ + links[id].name = devm_kasprintf(dev, GFP_KERNEL, + "SSP%d-Codec", ssp_codec); + if (!links[id].name) + goto devm_err; + + links[id].id = id; + links[id].codecs = pcm512x_component; + links[id].num_codecs = ARRAY_SIZE(pcm512x_component); + links[id].platforms = platform_component; + links[id].num_platforms = ARRAY_SIZE(platform_component); + links[id].init = sof_pcm512x_codec_init; + links[id].ops = &sof_pcm512x_ops; + links[id].nonatomic = true; + links[id].dpcm_playback = 1; + /* + * capture only supported with specific versions of the Hifiberry DAC+ + * links[id].dpcm_capture = 1; + */ + links[id].no_pcm = 1; + links[id].cpus = &cpus[id]; + links[id].num_cpus = 1; + if (is_legacy_cpu) { + links[id].cpus->dai_name = devm_kasprintf(dev, GFP_KERNEL, + "ssp%d-port", + ssp_codec); + if (!links[id].cpus->dai_name) + goto devm_err; + } else { + links[id].cpus->dai_name = devm_kasprintf(dev, GFP_KERNEL, + "SSP%d Pin", + ssp_codec); + if (!links[id].cpus->dai_name) + goto devm_err; + } + id++; + + /* dmic */ + if (dmic_be_num > 0) { + /* at least we have dmic01 */ + links[id].name = "dmic01"; + links[id].cpus = &cpus[id]; + links[id].cpus->dai_name = "DMIC01 Pin"; + links[id].init = dmic_init; + if (dmic_be_num > 1) { + /* set up 2 BE links at most */ + links[id + 1].name = "dmic16k"; + links[id + 1].cpus = &cpus[id + 1]; + links[id + 1].cpus->dai_name = "DMIC16k Pin"; + dmic_be_num = 2; + } + } + + for (i = 0; i < dmic_be_num; i++) { + links[id].id = id; + links[id].num_cpus = 1; + links[id].codecs = dmic_component; + links[id].num_codecs = ARRAY_SIZE(dmic_component); + links[id].platforms = platform_component; + links[id].num_platforms = ARRAY_SIZE(platform_component); + links[id].ignore_suspend = 1; + links[id].dpcm_capture = 1; + links[id].no_pcm = 1; + id++; + } + + /* HDMI */ + if (hdmi_num > 0) { + idisp_components = devm_kcalloc(dev, hdmi_num, + sizeof(struct snd_soc_dai_link_component), + GFP_KERNEL); + if (!idisp_components) + goto devm_err; + } + for (i = 1; i <= hdmi_num; i++) { + links[id].name = devm_kasprintf(dev, GFP_KERNEL, + "iDisp%d", i); + if (!links[id].name) + goto devm_err; + + links[id].id = id; + links[id].cpus = &cpus[id]; + links[id].num_cpus = 1; + links[id].cpus->dai_name = devm_kasprintf(dev, GFP_KERNEL, + "iDisp%d Pin", i); + if (!links[id].cpus->dai_name) + goto devm_err; + + /* + * topology cannot be loaded if codec is missing, so + * use the dummy codec if needed + */ + if (idisp_codec) { + idisp_components[i - 1].name = "ehdaudio0D2"; + idisp_components[i - 1].dai_name = + devm_kasprintf(dev, GFP_KERNEL, + "intel-hdmi-hifi%d", i); + } else { + idisp_components[i - 1].name = "snd-soc-dummy"; + idisp_components[i - 1].dai_name = "snd-soc-dummy-dai"; + } + if (!idisp_components[i - 1].dai_name) + goto devm_err; + + links[id].codecs = &idisp_components[i - 1]; + links[id].num_codecs = 1; + links[id].platforms = platform_component; + links[id].num_platforms = ARRAY_SIZE(platform_component); + links[id].init = sof_hdmi_init; + links[id].dpcm_playback = 1; + links[id].no_pcm = 1; + id++; + } + + return links; +devm_err: + return NULL; +} + +static int sof_audio_probe(struct platform_device *pdev) +{ + struct snd_soc_acpi_mach *mach = pdev->dev.platform_data; + struct snd_soc_dai_link *dai_links; + struct sof_card_private *ctx; + int dmic_be_num, hdmi_num; + int ret, ssp_codec; + + ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + hdmi_num = 0; + if (soc_intel_is_byt() || soc_intel_is_cht()) { + is_legacy_cpu = true; + dmic_be_num = 0; + /* default quirk for legacy cpu */ + sof_pcm512x_quirk = SOF_PCM512X_SSP_CODEC(2); + } else { + dmic_be_num = 2; +#if IS_ENABLED(CONFIG_SND_HDA_CODEC_HDMI) + if (mach->mach_params.common_hdmi_codec_drv && + (mach->mach_params.codec_mask & IDISP_CODEC_MASK)) + ctx->idisp_codec = true; + + /* links are always present in topology */ + hdmi_num = 3; +#endif + } + + dmi_check_system(sof_pcm512x_quirk_table); + + dev_dbg(&pdev->dev, "sof_pcm512x_quirk = %lx\n", sof_pcm512x_quirk); + + ssp_codec = sof_pcm512x_quirk & SOF_PCM512X_SSP_CODEC_MASK; + + /* compute number of dai links */ + sof_audio_card_pcm512x.num_links = 1 + dmic_be_num + hdmi_num; + + dai_links = sof_card_dai_links_create(&pdev->dev, ssp_codec, + dmic_be_num, hdmi_num, + ctx->idisp_codec); + if (!dai_links) + return -ENOMEM; + + sof_audio_card_pcm512x.dai_link = dai_links; + + INIT_LIST_HEAD(&ctx->hdmi_pcm_list); + + sof_audio_card_pcm512x.dev = &pdev->dev; + + /* set platform name for each dailink */ + ret = snd_soc_fixup_dai_links_platform_name(&sof_audio_card_pcm512x, + mach->mach_params.platform); + if (ret) + return ret; + + snd_soc_card_set_drvdata(&sof_audio_card_pcm512x, ctx); + + return devm_snd_soc_register_card(&pdev->dev, + &sof_audio_card_pcm512x); +} + +static int sof_pcm512x_remove(struct platform_device *pdev) +{ + struct snd_soc_card *card = platform_get_drvdata(pdev); + struct snd_soc_component *component = NULL; + + for_each_card_components(card, component) { + if (!strcmp(component->name, pcm512x_component[0].name)) { + snd_soc_component_set_jack(component, NULL, NULL); + break; + } + } + + return 0; +} + +static struct platform_driver sof_audio = { + .probe = sof_audio_probe, + .remove = sof_pcm512x_remove, + .driver = { + .name = "sof_pcm512x", + .pm = &snd_soc_pm_ops, + }, +}; +module_platform_driver(sof_audio) + +MODULE_DESCRIPTION("ASoC Intel(R) SOF + PCM512x Machine driver"); +MODULE_AUTHOR("Pierre-Louis Bossart"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:sof_pcm512x"); diff --git a/sound/soc/intel/boards/sof_rt5682.c b/sound/soc/intel/boards/sof_rt5682.c index 5d878873a8e0..8c29431b5847 100644 --- a/sound/soc/intel/boards/sof_rt5682.c +++ b/sound/soc/intel/boards/sof_rt5682.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 -// Copyright(c) 2019 Intel Corporation. +// Copyright(c) 2019-2020 Intel Corporation. /* * Intel SOF Machine Driver with Realtek rt5682 Codec - * and speaker codec MAX98357A + * and speaker codec MAX98357A or RT1015. */ #include #include @@ -18,10 +18,12 @@ #include #include #include +#include "../../codecs/rt1015.h" #include "../../codecs/rt5682.h" #include "../../codecs/hdac_hdmi.h" #include "../common/soc-intel-quirks.h" #include "hda_dsp_common.h" +#include "sof_maxim_common.h" #define NAME_SIZE 32 @@ -39,6 +41,8 @@ #define SOF_RT5682_NUM_HDMIDEV_MASK (GENMASK(12, 10)) #define SOF_RT5682_NUM_HDMIDEV(quirk) \ ((quirk << SOF_RT5682_NUM_HDMIDEV_SHIFT) & SOF_RT5682_NUM_HDMIDEV_MASK) +#define SOF_RT1015_SPEAKER_AMP_PRESENT BIT(13) +#define SOF_MAX98373_SPEAKER_AMP_PRESENT BIT(14) /* Default: MCLK on, MCLK 19.2M, SSP0 */ static unsigned long sof_rt5682_quirk = SOF_RT5682_MCLK_EN | @@ -120,7 +124,7 @@ static const struct dmi_system_id sof_rt5682_quirk_table[] = { static int sof_hdmi_init(struct snd_soc_pcm_runtime *rtd) { struct sof_card_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct sof_hdmi_pcm *pcm; pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); @@ -139,7 +143,7 @@ static int sof_hdmi_init(struct snd_soc_pcm_runtime *rtd) static int sof_rt5682_codec_init(struct snd_soc_pcm_runtime *rtd) { struct sof_card_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct snd_soc_jack *jack; int ret; @@ -207,7 +211,7 @@ static int sof_rt5682_hw_params(struct snd_pcm_substream *substream, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct sof_card_private *ctx = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int clk_id, clk_freq, pll_out, ret; if (sof_rt5682_quirk & SOF_RT5682_MCLK_EN) { @@ -260,6 +264,42 @@ static struct snd_soc_ops sof_rt5682_ops = { .hw_params = sof_rt5682_hw_params, }; +static int sof_rt1015_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_card *card = rtd->card; + struct snd_soc_dai *codec_dai; + int i, ret; + + if (!snd_soc_card_get_codec_dai(card, "rt1015-aif")) + return 0; + + for_each_rtd_codec_dais(rtd, i, codec_dai) { + ret = snd_soc_dai_set_pll(codec_dai, 0, RT1015_PLL_S_BCLK, + params_rate(params) * 50, + params_rate(params) * 256); + if (ret < 0) { + dev_err(card->dev, "failed to set pll\n"); + return ret; + } + /* Configure sysclk for codec */ + ret = snd_soc_dai_set_sysclk(codec_dai, RT1015_SCLK_S_PLL, + params_rate(params) * 256, + SND_SOC_CLOCK_IN); + if (ret < 0) { + dev_err(card->dev, "failed to set sysclk\n"); + return ret; + } + } + + return 0; +} + +static struct snd_soc_ops sof_rt1015_ops = { + .hw_params = sof_rt1015_hw_params, +}; + static struct snd_soc_dai_link_component platform_component[] = { { /* name might be overridden during probe */ @@ -316,12 +356,17 @@ static const struct snd_kcontrol_new sof_controls[] = { SOC_DAPM_PIN_SWITCH("Headphone Jack"), SOC_DAPM_PIN_SWITCH("Headset Mic"), SOC_DAPM_PIN_SWITCH("Spk"), + SOC_DAPM_PIN_SWITCH("Left Spk"), + SOC_DAPM_PIN_SWITCH("Right Spk"), + }; static const struct snd_soc_dapm_widget sof_widgets[] = { SND_SOC_DAPM_HP("Headphone Jack", NULL), SND_SOC_DAPM_MIC("Headset Mic", NULL), SND_SOC_DAPM_SPK("Spk", NULL), + SND_SOC_DAPM_SPK("Left Spk", NULL), + SND_SOC_DAPM_SPK("Right Spk", NULL), }; static const struct snd_soc_dapm_widget dmic_widgets[] = { @@ -342,11 +387,22 @@ static const struct snd_soc_dapm_route speaker_map[] = { { "Spk", NULL, "Speaker" }, }; +static const struct snd_soc_dapm_route speaker_map_lr[] = { + { "Left Spk", NULL, "Left SPO" }, + { "Right Spk", NULL, "Right SPO" }, +}; + static const struct snd_soc_dapm_route dmic_map[] = { /* digital mics */ {"DMic", NULL, "SoC DMIC"}, }; +static int speaker_codec_init_lr(struct snd_soc_pcm_runtime *rtd) +{ + return snd_soc_dapm_add_routes(&rtd->card->dapm, speaker_map_lr, + ARRAY_SIZE(speaker_map_lr)); +} + static int speaker_codec_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_card *card = rtd->card; @@ -382,6 +438,17 @@ static int dmic_init(struct snd_soc_pcm_runtime *rtd) return ret; } +static struct snd_soc_codec_conf rt1015_amp_conf[] = { + { + .dlc = COMP_CODEC_CONF("i2c-10EC1015:00"), + .name_prefix = "Left", + }, + { + .dlc = COMP_CODEC_CONF("i2c-10EC1015:01"), + .name_prefix = "Right", + }, +}; + /* sof audio machine driver for rt5682 codec */ static struct snd_soc_card sof_audio_card_rt5682 = { .name = "rt5682", /* the sof- prefix is added by the core */ @@ -417,6 +484,17 @@ static struct snd_soc_dai_link_component max98357a_component[] = { } }; +static struct snd_soc_dai_link_component rt1015_components[] = { + { + .name = "i2c-10EC1015:00", + .dai_name = "rt1015-aif", + }, + { + .name = "i2c-10EC1015:01", + .dai_name = "rt1015-aif", + }, +}; + static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, int ssp_codec, int ssp_amp, @@ -556,11 +634,24 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, goto devm_err; links[id].id = id; - links[id].codecs = max98357a_component; - links[id].num_codecs = ARRAY_SIZE(max98357a_component); + if (sof_rt5682_quirk & SOF_RT1015_SPEAKER_AMP_PRESENT) { + links[id].codecs = rt1015_components; + links[id].num_codecs = ARRAY_SIZE(rt1015_components); + links[id].init = speaker_codec_init_lr; + links[id].ops = &sof_rt1015_ops; + } else if (sof_rt5682_quirk & + SOF_MAX98373_SPEAKER_AMP_PRESENT) { + links[id].codecs = max_98373_components; + links[id].num_codecs = ARRAY_SIZE(max_98373_components); + links[id].init = max98373_spk_codec_init; + links[id].ops = &max_98373_ops; + } else { + links[id].codecs = max98357a_component; + links[id].num_codecs = ARRAY_SIZE(max98357a_component); + links[id].init = speaker_codec_init; + } links[id].platforms = platform_component; links[id].num_platforms = ARRAY_SIZE(platform_component); - links[id].init = speaker_codec_init, links[id].nonatomic = true; links[id].dpcm_playback = 1; links[id].no_pcm = 1; @@ -604,7 +695,7 @@ static int sof_audio_probe(struct platform_device *pdev) dmi_check_system(sof_rt5682_quirk_table); - mach = (&pdev->dev)->platform_data; + mach = pdev->dev.platform_data; /* A speaker amp might not be present when the quirk claims one is. * Detect this via whether the machine driver match includes quirk_data. @@ -662,6 +753,9 @@ static int sof_audio_probe(struct platform_device *pdev) if (sof_rt5682_quirk & SOF_SPEAKER_AMP_PRESENT) sof_audio_card_rt5682.num_links++; + if (sof_rt5682_quirk & SOF_MAX98373_SPEAKER_AMP_PRESENT) + sof_max98373_codec_conf(&sof_audio_card_rt5682); + dai_links = sof_card_dai_links_create(&pdev->dev, ssp_codec, ssp_amp, dmic_be_num, hdmi_num); if (!dai_links) @@ -669,6 +763,11 @@ static int sof_audio_probe(struct platform_device *pdev) sof_audio_card_rt5682.dai_link = dai_links; + if (sof_rt5682_quirk & SOF_RT1015_SPEAKER_AMP_PRESENT) { + sof_audio_card_rt5682.codec_conf = rt1015_amp_conf; + sof_audio_card_rt5682.num_configs = ARRAY_SIZE(rt1015_amp_conf); + } + INIT_LIST_HEAD(&ctx->hdmi_pcm_list); sof_audio_card_rt5682.dev = &pdev->dev; @@ -714,6 +813,24 @@ static const struct platform_device_id board_ids[] = { SOF_RT5682_SSP_AMP(1) | SOF_RT5682_NUM_HDMIDEV(4)), }, + { + .name = "jsl_rt5682_rt1015", + .driver_data = (kernel_ulong_t)(SOF_RT5682_MCLK_EN | + SOF_RT5682_MCLK_24MHZ | + SOF_RT5682_SSP_CODEC(0) | + SOF_SPEAKER_AMP_PRESENT | + SOF_RT1015_SPEAKER_AMP_PRESENT | + SOF_RT5682_SSP_AMP(1)), + }, + { + .name = "tgl_max98373_rt5682", + .driver_data = (kernel_ulong_t)(SOF_RT5682_MCLK_EN | + SOF_RT5682_SSP_CODEC(0) | + SOF_SPEAKER_AMP_PRESENT | + SOF_MAX98373_SPEAKER_AMP_PRESENT | + SOF_RT5682_SSP_AMP(1) | + SOF_RT5682_NUM_HDMIDEV(4)), + }, { } }; @@ -735,3 +852,5 @@ MODULE_AUTHOR("Sathya Prakash M R "); MODULE_LICENSE("GPL v2"); MODULE_ALIAS("platform:sof_rt5682"); MODULE_ALIAS("platform:tgl_max98357a_rt5682"); +MODULE_ALIAS("platform:jsl_rt5682_rt1015"); +MODULE_ALIAS("platform:tgl_max98373_rt5682"); diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c new file mode 100644 index 000000000000..a64dc563b47e --- /dev/null +++ b/sound/soc/intel/boards/sof_sdw.c @@ -0,0 +1,962 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Intel Corporation + +/* + * sof_sdw - ASOC Machine driver for Intel SoundWire platforms + */ + +#include +#include +#include +#include +#include +#include +#include +#include "sof_sdw_common.h" + +unsigned long sof_sdw_quirk = SOF_RT711_JD_SRC_JD1; + +#define INC_ID(BE, CPU, LINK) do { (BE)++; (CPU)++; (LINK)++; } while (0) + +static int sof_sdw_quirk_cb(const struct dmi_system_id *id) +{ + sof_sdw_quirk = (unsigned long)id->driver_data; + return 1; +} + +static const struct dmi_system_id sof_sdw_quirk_table[] = { + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "09C6") + }, + .driver_data = (void *)(SOF_RT711_JD_SRC_JD2 | + SOF_RT715_DAI_ID_FIX), + }, + { + /* early version of SKU 09C6 */ + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0983") + }, + .driver_data = (void *)(SOF_RT711_JD_SRC_JD2 | + SOF_RT715_DAI_ID_FIX), + }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "098F"), + }, + .driver_data = (void *)(SOF_RT711_JD_SRC_JD2 | + SOF_RT715_DAI_ID_FIX | + SOF_SDW_FOUR_SPK), + }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0990"), + }, + .driver_data = (void *)(SOF_RT711_JD_SRC_JD2 | + SOF_RT715_DAI_ID_FIX | + SOF_SDW_FOUR_SPK), + }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Intel Corporation"), + DMI_MATCH(DMI_PRODUCT_NAME, + "Tiger Lake Client Platform"), + }, + .driver_data = (void *)(SOF_RT711_JD_SRC_JD1 | + SOF_SDW_TGL_HDMI | SOF_SDW_PCH_DMIC | + SOF_SSP_PORT(SOF_I2S_SSP2)), + }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Intel Corporation"), + DMI_MATCH(DMI_PRODUCT_NAME, "Ice Lake Client"), + }, + .driver_data = (void *)SOF_SDW_PCH_DMIC, + }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Intel Corporation"), + DMI_MATCH(DMI_PRODUCT_NAME, "CometLake Client"), + }, + .driver_data = (void *)SOF_SDW_PCH_DMIC, + }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Google"), + DMI_MATCH(DMI_PRODUCT_NAME, "Volteer"), + }, + .driver_data = (void *)(SOF_SDW_TGL_HDMI | SOF_SDW_PCH_DMIC), + }, + + {} +}; + +static struct snd_soc_codec_conf codec_conf[] = { + { + .dlc = COMP_CODEC_CONF("sdw:0:25d:711:0"), + .name_prefix = "rt711", + }, + /* rt1308 w/ I2S connection */ + { + .dlc = COMP_CODEC_CONF("i2c-10EC1308:00"), + .name_prefix = "rt1308-1", + }, + /* rt1308 left on link 1 */ + { + .dlc = COMP_CODEC_CONF("sdw:1:25d:1308:0"), + .name_prefix = "rt1308-1", + }, + /* two 1308s on link1 with different unique id */ + { + .dlc = COMP_CODEC_CONF("sdw:1:25d:1308:0:0"), + .name_prefix = "rt1308-1", + }, + { + .dlc = COMP_CODEC_CONF("sdw:1:25d:1308:0:2"), + .name_prefix = "rt1308-2", + }, + /* rt1308 right on link 2 */ + { + .dlc = COMP_CODEC_CONF("sdw:2:25d:1308:0"), + .name_prefix = "rt1308-2", + }, + { + .dlc = COMP_CODEC_CONF("sdw:3:25d:715:0"), + .name_prefix = "rt715", + }, + { + .dlc = COMP_CODEC_CONF("sdw:0:25d:5682:0"), + .name_prefix = "rt5682", + }, +}; + +static struct snd_soc_dai_link_component dmic_component[] = { + { + .name = "dmic-codec", + .dai_name = "dmic-hifi", + } +}; + +static struct snd_soc_dai_link_component platform_component[] = { + { + /* name might be overridden during probe */ + .name = "0000:00:1f.3" + } +}; + +/* these wrappers are only needed to avoid typecast compilation errors */ +static int sdw_startup(struct snd_pcm_substream *substream) +{ + return sdw_startup_stream(substream); +} + +static void sdw_shutdown(struct snd_pcm_substream *substream) +{ + sdw_shutdown_stream(substream); +} + +static const struct snd_soc_ops sdw_ops = { + .startup = sdw_startup, + .shutdown = sdw_shutdown, +}; + +static struct sof_sdw_codec_info codec_info_list[] = { + { + .id = 0x700, + .direction = {true, true}, + .dai_name = "rt700-aif1", + .init = sof_sdw_rt700_init, + }, + { + .id = 0x711, + .direction = {true, true}, + .dai_name = "rt711-aif1", + .init = sof_sdw_rt711_init, + }, + { + .id = 0x1308, + .acpi_id = "10EC1308", + .direction = {true, false}, + .dai_name = "rt1308-aif", + .ops = &sof_sdw_rt1308_i2s_ops, + .init = sof_sdw_rt1308_init, + }, + { + .id = 0x715, + .direction = {false, true}, + .dai_name = "rt715-aif2", + .init = sof_sdw_rt715_init, + }, + { + .id = 0x5682, + .direction = {true, true}, + .dai_name = "rt5682-sdw", + .init = sof_sdw_rt5682_init, + }, +}; + +static inline int find_codec_info_part(unsigned int part_id) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(codec_info_list); i++) + if (part_id == codec_info_list[i].id) + break; + + if (i == ARRAY_SIZE(codec_info_list)) + return -EINVAL; + + return i; +} + +static inline int find_codec_info_acpi(const u8 *acpi_id) +{ + int i; + + if (!acpi_id[0]) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(codec_info_list); i++) + if (!memcmp(codec_info_list[i].acpi_id, acpi_id, + ACPI_ID_LEN)) + break; + + if (i == ARRAY_SIZE(codec_info_list)) + return -EINVAL; + + return i; +} + +/* + * get BE dailink number and CPU DAI number based on sdw link adr. + * Since some sdw slaves may be aggregated, the CPU DAI number + * may be larger than the number of BE dailinks. + */ +static int get_sdw_dailink_info(const struct snd_soc_acpi_link_adr *links, + int *sdw_be_num, int *sdw_cpu_dai_num) +{ + const struct snd_soc_acpi_link_adr *link; + bool group_visited[SDW_MAX_GROUPS]; + bool no_aggregation; + int i; + + no_aggregation = sof_sdw_quirk & SOF_SDW_NO_AGGREGATION; + *sdw_cpu_dai_num = 0; + *sdw_be_num = 0; + + if (!links) + return -EINVAL; + + for (i = 0; i < SDW_MAX_GROUPS; i++) + group_visited[i] = false; + + for (link = links; link->num_adr; link++) { + const struct snd_soc_acpi_endpoint *endpoint; + int part_id, codec_index; + int stream; + u64 adr; + + adr = link->adr_d->adr; + part_id = SDW_PART_ID(adr); + codec_index = find_codec_info_part(part_id); + if (codec_index < 0) + return codec_index; + + endpoint = link->adr_d->endpoints; + + /* count DAI number for playback and capture */ + for_each_pcm_streams(stream) { + if (!codec_info_list[codec_index].direction[stream]) + continue; + + (*sdw_cpu_dai_num)++; + + /* count BE for each non-aggregated slave or group */ + if (!endpoint->aggregated || no_aggregation || + !group_visited[endpoint->group_id]) + (*sdw_be_num)++; + } + + if (endpoint->aggregated) + group_visited[endpoint->group_id] = true; + } + + return 0; +} + +static void init_dai_link(struct snd_soc_dai_link *dai_links, int be_id, + char *name, int playback, int capture, + struct snd_soc_dai_link_component *cpus, + int cpus_num, + struct snd_soc_dai_link_component *codecs, + int codecs_num, + int (*init)(struct snd_soc_pcm_runtime *rtd), + const struct snd_soc_ops *ops) +{ + dai_links->id = be_id; + dai_links->name = name; + dai_links->platforms = platform_component; + dai_links->num_platforms = ARRAY_SIZE(platform_component); + dai_links->nonatomic = true; + dai_links->no_pcm = 1; + dai_links->cpus = cpus; + dai_links->num_cpus = cpus_num; + dai_links->codecs = codecs; + dai_links->num_codecs = codecs_num; + dai_links->dpcm_playback = playback; + dai_links->dpcm_capture = capture; + dai_links->init = init; + dai_links->ops = ops; +} + +static bool is_unique_device(const struct snd_soc_acpi_link_adr *link, + unsigned int sdw_version, + unsigned int mfg_id, + unsigned int part_id, + unsigned int class_id, + int index_in_link + ) +{ + int i; + + for (i = 0; i < link->num_adr; i++) { + unsigned int sdw1_version, mfg1_id, part1_id, class1_id; + u64 adr; + + /* skip itself */ + if (i == index_in_link) + continue; + + adr = link->adr_d[i].adr; + + sdw1_version = SDW_VERSION(adr); + mfg1_id = SDW_MFG_ID(adr); + part1_id = SDW_PART_ID(adr); + class1_id = SDW_CLASS_ID(adr); + + if (sdw_version == sdw1_version && + mfg_id == mfg1_id && + part_id == part1_id && + class_id == class1_id) + return false; + } + + return true; +} + +static int create_codec_dai_name(struct device *dev, + const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link_component *codec, + int offset) +{ + int i; + + for (i = 0; i < link->num_adr; i++) { + unsigned int sdw_version, unique_id, mfg_id; + unsigned int link_id, part_id, class_id; + int codec_index, comp_index; + char *codec_str; + u64 adr; + + adr = link->adr_d[i].adr; + + sdw_version = SDW_VERSION(adr); + link_id = SDW_DISCO_LINK_ID(adr); + unique_id = SDW_UNIQUE_ID(adr); + mfg_id = SDW_MFG_ID(adr); + part_id = SDW_PART_ID(adr); + class_id = SDW_CLASS_ID(adr); + + comp_index = i + offset; + if (is_unique_device(link, sdw_version, mfg_id, part_id, + class_id, i)) { + codec_str = "sdw:%x:%x:%x:%x"; + codec[comp_index].name = + devm_kasprintf(dev, GFP_KERNEL, codec_str, + link_id, mfg_id, part_id, + class_id); + } else { + codec_str = "sdw:%x:%x:%x:%x:%x"; + codec[comp_index].name = + devm_kasprintf(dev, GFP_KERNEL, codec_str, + link_id, mfg_id, part_id, + class_id, unique_id); + } + + if (!codec[comp_index].name) + return -ENOMEM; + + codec_index = find_codec_info_part(part_id); + if (codec_index < 0) + return codec_index; + + codec[comp_index].dai_name = + codec_info_list[codec_index].dai_name; + } + + return 0; +} + +static int set_codec_init_func(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + bool playback) +{ + int i; + + for (i = 0; i < link->num_adr; i++) { + unsigned int part_id; + int codec_index; + + part_id = SDW_PART_ID(link->adr_d[i].adr); + codec_index = find_codec_info_part(part_id); + + if (codec_index < 0) + return codec_index; + + if (codec_info_list[codec_index].init) + codec_info_list[codec_index].init(link, dai_links, + &codec_info_list[codec_index], + playback); + } + + return 0; +} + +/* + * check endpoint status in slaves and gather link ID for all slaves in + * the same group to generate different CPU DAI. Now only support + * one sdw link with all slaves set with only single group id. + * + * one slave on one sdw link with aggregated = 0 + * one sdw BE DAI <---> one-cpu DAI <---> one-codec DAI + * + * two or more slaves on one sdw link with aggregated = 0 + * one sdw BE DAI <---> one-cpu DAI <---> multi-codec DAIs + * + * multiple links with multiple slaves with aggregated = 1 + * one sdw BE DAI <---> 1 .. N CPU DAIs <----> 1 .. N codec DAIs + */ +static int get_slave_info(const struct snd_soc_acpi_link_adr *adr_link, + struct device *dev, int *cpu_dai_id, int *cpu_dai_num, + int *codec_num, int *group_id, + bool *group_generated) +{ + const struct snd_soc_acpi_adr_device *adr_d; + const struct snd_soc_acpi_link_adr *adr_next; + bool no_aggregation; + int index = 0; + + no_aggregation = sof_sdw_quirk & SOF_SDW_NO_AGGREGATION; + *codec_num = adr_link->num_adr; + adr_d = adr_link->adr_d; + + /* make sure the link mask has a single bit set */ + if (!is_power_of_2(adr_link->mask)) + return -EINVAL; + + cpu_dai_id[index++] = ffs(adr_link->mask) - 1; + if (!adr_d->endpoints->aggregated || no_aggregation) { + *cpu_dai_num = 1; + *group_id = 0; + return 0; + } + + *group_id = adr_d->endpoints->group_id; + + /* gather other link ID of slaves in the same group */ + for (adr_next = adr_link + 1; adr_next && adr_next->num_adr; + adr_next++) { + const struct snd_soc_acpi_endpoint *endpoint; + + endpoint = adr_next->adr_d->endpoints; + if (!endpoint->aggregated || + endpoint->group_id != *group_id) + continue; + + /* make sure the link mask has a single bit set */ + if (!is_power_of_2(adr_next->mask)) + return -EINVAL; + + if (index >= SDW_MAX_CPU_DAIS) { + dev_err(dev, " cpu_dai_id array overflows"); + return -EINVAL; + } + + cpu_dai_id[index++] = ffs(adr_next->mask) - 1; + *codec_num += adr_next->num_adr; + } + + /* + * indicate CPU DAIs for this group have been generated + * to avoid generating CPU DAIs for this group again. + */ + group_generated[*group_id] = true; + *cpu_dai_num = index; + + return 0; +} + +static int create_sdw_dailink(struct device *dev, int *be_index, + struct snd_soc_dai_link *dai_links, + int sdw_be_num, int sdw_cpu_dai_num, + struct snd_soc_dai_link_component *cpus, + const struct snd_soc_acpi_link_adr *link, + int *cpu_id, bool *group_generated) +{ + const struct snd_soc_acpi_link_adr *link_next; + struct snd_soc_dai_link_component *codecs; + int cpu_dai_id[SDW_MAX_CPU_DAIS]; + int cpu_dai_num, cpu_dai_index; + unsigned int part_id, group_id; + int codec_idx = 0; + int i = 0, j = 0; + int codec_index; + int codec_num; + int stream; + int ret; + int k; + + ret = get_slave_info(link, dev, cpu_dai_id, &cpu_dai_num, &codec_num, + &group_id, group_generated); + if (ret) + return ret; + + codecs = devm_kcalloc(dev, codec_num, sizeof(*codecs), GFP_KERNEL); + if (!codecs) + return -ENOMEM; + + /* generate codec name on different links in the same group */ + for (link_next = link; link_next && link_next->num_adr && + i < cpu_dai_num; link_next++) { + const struct snd_soc_acpi_endpoint *endpoints; + + endpoints = link_next->adr_d->endpoints; + if (group_id && (!endpoints->aggregated || + endpoints->group_id != group_id)) + continue; + + /* skip the link excluded by this processed group */ + if (cpu_dai_id[i] != ffs(link_next->mask) - 1) + continue; + + ret = create_codec_dai_name(dev, link_next, codecs, codec_idx); + if (ret < 0) + return ret; + + /* check next link to create codec dai in the processed group */ + i++; + codec_idx += link_next->num_adr; + } + + /* find codec info to create BE DAI */ + part_id = SDW_PART_ID(link->adr_d[0].adr); + codec_index = find_codec_info_part(part_id); + if (codec_index < 0) + return codec_index; + + cpu_dai_index = *cpu_id; + for_each_pcm_streams(stream) { + char *name, *cpu_name; + int playback, capture; + static const char * const sdw_stream_name[] = { + "SDW%d-Playback", + "SDW%d-Capture", + }; + + if (!codec_info_list[codec_index].direction[stream]) + continue; + + /* create stream name according to first link id */ + name = devm_kasprintf(dev, GFP_KERNEL, + sdw_stream_name[stream], cpu_dai_id[0]); + if (!name) + return -ENOMEM; + + /* + * generate CPU DAI name base on the sdw link ID and + * PIN ID with offset of 2 according to sdw dai driver. + */ + for (k = 0; k < cpu_dai_num; k++) { + cpu_name = devm_kasprintf(dev, GFP_KERNEL, + "SDW%d Pin%d", cpu_dai_id[k], + j + SDW_INTEL_BIDIR_PDI_BASE); + if (!cpu_name) + return -ENOMEM; + + if (cpu_dai_index >= sdw_cpu_dai_num) { + dev_err(dev, "invalid cpu dai index %d", + cpu_dai_index); + return -EINVAL; + } + + cpus[cpu_dai_index++].dai_name = cpu_name; + } + + if (*be_index >= sdw_be_num) { + dev_err(dev, " invalid be dai index %d", *be_index); + return -EINVAL; + } + + if (*cpu_id >= sdw_cpu_dai_num) { + dev_err(dev, " invalid cpu dai index %d", *cpu_id); + return -EINVAL; + } + + playback = (stream == SNDRV_PCM_STREAM_PLAYBACK); + capture = (stream == SNDRV_PCM_STREAM_CAPTURE); + init_dai_link(dai_links + *be_index, *be_index, name, + playback, capture, + cpus + *cpu_id, cpu_dai_num, + codecs, codec_num, + NULL, &sdw_ops); + + ret = set_codec_init_func(link, dai_links + (*be_index)++, + playback); + if (ret < 0) { + dev_err(dev, "failed to init codec %d", codec_index); + return ret; + } + + *cpu_id += cpu_dai_num; + j++; + } + + return 0; +} + +/* + * DAI link ID of SSP & DMIC & HDMI are based on last + * link ID used by sdw link. Since be_id may be changed + * in init func of sdw codec, it is not equal to be_id + */ +static inline int get_next_be_id(struct snd_soc_dai_link *links, + int be_id) +{ + return links[be_id - 1].id + 1; +} + +static int sof_card_dai_links_create(struct device *dev, + struct snd_soc_acpi_mach *mach, + struct snd_soc_card *card) +{ + int ssp_num, sdw_be_num = 0, hdmi_num = 0, dmic_num; +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC) + struct snd_soc_dai_link_component *idisp_components; +#endif + struct snd_soc_dai_link_component *ssp_components; + struct snd_soc_acpi_mach_params *mach_params; + const struct snd_soc_acpi_link_adr *adr_link; + struct snd_soc_dai_link_component *cpus; + bool group_generated[SDW_MAX_GROUPS]; + int ssp_codec_index, ssp_mask; + struct snd_soc_dai_link *links; + int num_links, link_id = 0; + char *name, *cpu_name; + int total_cpu_dai_num; + int sdw_cpu_dai_num; + int i, j, be_id = 0; + int cpu_id = 0; + int comp_num; + int ret; + + /* reset amp_num to ensure amp_num++ starts from 0 in each probe */ + for (i = 0; i < ARRAY_SIZE(codec_info_list); i++) + codec_info_list[i].amp_num = 0; + +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC) + hdmi_num = sof_sdw_quirk & SOF_SDW_TGL_HDMI ? + SOF_TGL_HDMI_COUNT : SOF_PRE_TGL_HDMI_COUNT; +#endif + + ssp_mask = SOF_SSP_GET_PORT(sof_sdw_quirk); + /* + * on generic tgl platform, I2S or sdw mode is supported + * based on board rework. A ACPI device is registered in + * system only when I2S mode is supported, not sdw mode. + * Here check ACPI ID to confirm I2S is supported. + */ + ssp_codec_index = find_codec_info_acpi(mach->id); + ssp_num = ssp_codec_index >= 0 ? hweight_long(ssp_mask) : 0; + comp_num = hdmi_num + ssp_num; + + mach_params = &mach->mach_params; + ret = get_sdw_dailink_info(mach_params->links, + &sdw_be_num, &sdw_cpu_dai_num); + if (ret < 0) { + dev_err(dev, "failed to get sdw link info %d", ret); + return ret; + } + + /* enable dmic01 & dmic16k */ + dmic_num = (sof_sdw_quirk & SOF_SDW_PCH_DMIC) ? 2 : 0; + comp_num += dmic_num; + + dev_dbg(dev, "sdw %d, ssp %d, dmic %d, hdmi %d", sdw_be_num, ssp_num, + dmic_num, hdmi_num); + + /* allocate BE dailinks */ + num_links = comp_num + sdw_be_num; + links = devm_kcalloc(dev, num_links, sizeof(*links), GFP_KERNEL); + + /* allocated CPU DAIs */ + total_cpu_dai_num = comp_num + sdw_cpu_dai_num; + cpus = devm_kcalloc(dev, total_cpu_dai_num, sizeof(*cpus), + GFP_KERNEL); + + if (!links || !cpus) + return -ENOMEM; + + /* SDW */ + if (!sdw_be_num) + goto SSP; + + adr_link = mach_params->links; + if (!adr_link) + return -EINVAL; + + /* + * SoundWire Slaves aggregated in the same group may be + * located on different hardware links. Clear array to indicate + * CPU DAIs for this group have not been generated. + */ + for (i = 0; i < SDW_MAX_GROUPS; i++) + group_generated[i] = false; + + /* generate DAI links by each sdw link */ + for (; adr_link->num_adr; adr_link++) { + const struct snd_soc_acpi_endpoint *endpoint; + + endpoint = adr_link->adr_d->endpoints; + if (endpoint->aggregated && !endpoint->group_id) { + dev_err(dev, "invalid group id on link %x", + adr_link->mask); + continue; + } + + /* this group has been generated */ + if (endpoint->aggregated && + group_generated[endpoint->group_id]) + continue; + + ret = create_sdw_dailink(dev, &be_id, links, sdw_be_num, + sdw_cpu_dai_num, cpus, adr_link, + &cpu_id, group_generated); + if (ret < 0) { + dev_err(dev, "failed to create dai link %d", be_id); + return -ENOMEM; + } + } + + /* non-sdw DAI follows sdw DAI */ + link_id = be_id; + + /* get BE ID for non-sdw DAI */ + be_id = get_next_be_id(links, be_id); + +SSP: + /* SSP */ + if (!ssp_num) + goto DMIC; + + for (i = 0, j = 0; ssp_mask; i++, ssp_mask >>= 1) { + struct sof_sdw_codec_info *info; + int playback, capture; + char *codec_name; + + if (!(ssp_mask & 0x1)) + continue; + + name = devm_kasprintf(dev, GFP_KERNEL, + "SSP%d-Codec", i); + if (!name) + return -ENOMEM; + + cpu_name = devm_kasprintf(dev, GFP_KERNEL, "SSP%d Pin", i); + if (!cpu_name) + return -ENOMEM; + + ssp_components = devm_kzalloc(dev, sizeof(*ssp_components), + GFP_KERNEL); + if (!ssp_components) + return -ENOMEM; + + info = &codec_info_list[ssp_codec_index]; + codec_name = devm_kasprintf(dev, GFP_KERNEL, "i2c-%s:0%d", + info->acpi_id, j++); + if (!codec_name) + return -ENOMEM; + + ssp_components->name = codec_name; + ssp_components->dai_name = info->dai_name; + cpus[cpu_id].dai_name = cpu_name; + + playback = info->direction[SNDRV_PCM_STREAM_PLAYBACK]; + capture = info->direction[SNDRV_PCM_STREAM_CAPTURE]; + init_dai_link(links + link_id, be_id, name, + playback, capture, + cpus + cpu_id, 1, + ssp_components, 1, + NULL, info->ops); + + ret = info->init(NULL, links + link_id, info, 0); + if (ret < 0) + return ret; + + INC_ID(be_id, cpu_id, link_id); + } + +DMIC: + /* dmic */ + if (dmic_num > 0) { + cpus[cpu_id].dai_name = "DMIC01 Pin"; + init_dai_link(links + link_id, be_id, "dmic01", + 0, 1, // DMIC only supports capture + cpus + cpu_id, 1, + dmic_component, 1, + sof_sdw_dmic_init, NULL); + INC_ID(be_id, cpu_id, link_id); + + cpus[cpu_id].dai_name = "DMIC16k Pin"; + init_dai_link(links + link_id, be_id, "dmic16k", + 0, 1, // DMIC only supports capture + cpus + cpu_id, 1, + dmic_component, 1, + /* don't call sof_sdw_dmic_init() twice */ + NULL, NULL); + INC_ID(be_id, cpu_id, link_id); + } + +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC) + /* HDMI */ + if (hdmi_num > 0) { + idisp_components = devm_kcalloc(dev, hdmi_num, + sizeof(*idisp_components), + GFP_KERNEL); + if (!idisp_components) + return -ENOMEM; + } + + for (i = 0; i < hdmi_num; i++) { + name = devm_kasprintf(dev, GFP_KERNEL, + "iDisp%d", i + 1); + if (!name) + return -ENOMEM; + + idisp_components[i].name = "ehdaudio0D2"; + idisp_components[i].dai_name = devm_kasprintf(dev, + GFP_KERNEL, + "intel-hdmi-hifi%d", + i + 1); + if (!idisp_components[i].dai_name) + return -ENOMEM; + + cpu_name = devm_kasprintf(dev, GFP_KERNEL, + "iDisp%d Pin", i + 1); + if (!cpu_name) + return -ENOMEM; + + cpus[cpu_id].dai_name = cpu_name; + init_dai_link(links + link_id, be_id, name, + 1, 0, // HDMI only supports playback + cpus + cpu_id, 1, + idisp_components + i, 1, + sof_sdw_hdmi_init, NULL); + INC_ID(be_id, cpu_id, link_id); + } +#endif + + card->dai_link = links; + card->num_links = num_links; + + return 0; +} + +/* SoC card */ +static const char sdw_card_long_name[] = "Intel Soundwire SOF"; + +static struct snd_soc_card card_sof_sdw = { + .name = "soundwire", + .late_probe = sof_sdw_hdmi_card_late_probe, + .codec_conf = codec_conf, + .num_configs = ARRAY_SIZE(codec_conf), +}; + +static int mc_probe(struct platform_device *pdev) +{ + struct snd_soc_card *card = &card_sof_sdw; + struct snd_soc_acpi_mach *mach; + struct mc_private *ctx; + int ret; + + dev_dbg(&pdev->dev, "Entry %s\n", __func__); + + ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + dmi_check_system(sof_sdw_quirk_table); + +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC) + INIT_LIST_HEAD(&ctx->hdmi_pcm_list); +#endif + + card->dev = &pdev->dev; + + mach = pdev->dev.platform_data; + ret = sof_card_dai_links_create(&pdev->dev, mach, + card); + if (ret < 0) + return ret; + + ctx->common_hdmi_codec_drv = mach->mach_params.common_hdmi_codec_drv; + + snd_soc_card_set_drvdata(card, ctx); + + card->components = devm_kasprintf(card->dev, GFP_KERNEL, + "cfg-spk:%d", + (sof_sdw_quirk & SOF_SDW_FOUR_SPK) ? 4 : 2); + if (!card->components) + return -ENOMEM; + + card->long_name = sdw_card_long_name; + + /* Register the card */ + ret = devm_snd_soc_register_card(&pdev->dev, card); + if (ret) { + dev_err(card->dev, "snd_soc_register_card failed %d\n", ret); + return ret; + } + + platform_set_drvdata(pdev, card); + + return ret; +} + +static struct platform_driver sof_sdw_driver = { + .driver = { + .name = "sof_sdw", + .pm = &snd_soc_pm_ops, + }, + .probe = mc_probe, +}; + +module_platform_driver(sof_sdw_driver); + +MODULE_DESCRIPTION("ASoC SoundWire Generic Machine driver"); +MODULE_AUTHOR("Bard Liao "); +MODULE_AUTHOR("Rander Wang "); +MODULE_AUTHOR("Pierre-Louis Bossart "); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:sof_sdw"); diff --git a/sound/soc/intel/boards/sof_sdw_common.h b/sound/soc/intel/boards/sof_sdw_common.h new file mode 100644 index 000000000000..dd593ff3575b --- /dev/null +++ b/sound/soc/intel/boards/sof_sdw_common.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Copyright (c) 2020 Intel Corporation + */ + +/* + * sof_sdw_common.h - prototypes for common helpers + */ + +#ifndef SND_SOC_SOF_SDW_COMMON_H +#define SND_SOC_SOF_SDW_COMMON_H + +#include +#include + +#define MAX_NO_PROPS 2 +#define MAX_HDMI_NUM 4 +#define SDW_DMIC_DAI_ID 4 +#define SDW_MAX_CPU_DAIS 16 +#define SDW_INTEL_BIDIR_PDI_BASE 2 + +/* 8 combinations with 4 links + unused group 0 */ +#define SDW_MAX_GROUPS 9 + +enum { + SOF_RT711_JD_SRC_JD1 = 1, + SOF_RT711_JD_SRC_JD2 = 2, +}; + +enum { + SOF_PRE_TGL_HDMI_COUNT = 3, + SOF_TGL_HDMI_COUNT = 4, +}; + +enum { + SOF_I2S_SSP0 = BIT(0), + SOF_I2S_SSP1 = BIT(1), + SOF_I2S_SSP2 = BIT(2), + SOF_I2S_SSP3 = BIT(3), + SOF_I2S_SSP4 = BIT(4), + SOF_I2S_SSP5 = BIT(5), +}; + +#define SOF_RT711_JDSRC(quirk) ((quirk) & GENMASK(1, 0)) +#define SOF_SDW_FOUR_SPK BIT(2) +#define SOF_SDW_TGL_HDMI BIT(3) +#define SOF_SDW_PCH_DMIC BIT(4) +#define SOF_SSP_PORT(x) (((x) & GENMASK(5, 0)) << 5) +#define SOF_SSP_GET_PORT(quirk) (((quirk) >> 5) & GENMASK(5, 0)) +#define SOF_RT715_DAI_ID_FIX BIT(11) +#define SOF_SDW_NO_AGGREGATION BIT(12) + +struct sof_sdw_codec_info { + const int id; + int amp_num; + const u8 acpi_id[ACPI_ID_LEN]; + const bool direction[2]; // playback & capture support + const char *dai_name; + const struct snd_soc_ops *ops; + + int (*init)(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback); +}; + +struct mc_private { + struct list_head hdmi_pcm_list; + bool common_hdmi_codec_drv; + struct snd_soc_jack sdw_headset; +}; + +extern unsigned long sof_sdw_quirk; + +/* generic HDMI support */ +int sof_sdw_hdmi_init(struct snd_soc_pcm_runtime *rtd); + +int sof_sdw_hdmi_card_late_probe(struct snd_soc_card *card); + +/* DMIC support */ +int sof_sdw_dmic_init(struct snd_soc_pcm_runtime *rtd); + +/* RT711 support */ +int sof_sdw_rt711_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback); + +/* RT700 support */ +int sof_sdw_rt700_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback); + +/* RT1308 support */ +extern struct snd_soc_ops sof_sdw_rt1308_i2s_ops; + +int sof_sdw_rt1308_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback); + +/* RT715 support */ +int sof_sdw_rt715_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback); + +/* RT5682 support */ +int sof_sdw_rt5682_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback); + +#endif diff --git a/sound/soc/intel/boards/sof_sdw_dmic.c b/sound/soc/intel/boards/sof_sdw_dmic.c new file mode 100644 index 000000000000..e92176bf0ad4 --- /dev/null +++ b/sound/soc/intel/boards/sof_sdw_dmic.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Intel Corporation + +/* + * sof_sdw_dmic - Helpers to handle dmic from generic machine driver + */ + +#include +#include +#include "sof_sdw_common.h" + +static const struct snd_soc_dapm_widget dmic_widgets[] = { + SND_SOC_DAPM_MIC("SoC DMIC", NULL), +}; + +static const struct snd_soc_dapm_route dmic_map[] = { + /* digital mics */ + {"DMic", NULL, "SoC DMIC"}, +}; + +int sof_sdw_dmic_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + int ret; + + ret = snd_soc_dapm_new_controls(&card->dapm, dmic_widgets, + ARRAY_SIZE(dmic_widgets)); + if (ret) { + dev_err(card->dev, "DMic widget addition failed: %d\n", ret); + /* Don't need to add routes if widget addition failed */ + return ret; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, dmic_map, + ARRAY_SIZE(dmic_map)); + + if (ret) + dev_err(card->dev, "DMic map addition failed: %d\n", ret); + + return ret; +} + diff --git a/sound/soc/intel/boards/sof_sdw_hdmi.c b/sound/soc/intel/boards/sof_sdw_hdmi.c new file mode 100644 index 000000000000..c7b5612a39e6 --- /dev/null +++ b/sound/soc/intel/boards/sof_sdw_hdmi.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Intel Corporation + +/* + * sof_sdw_hdmi - Helpers to handle HDMI from generic machine driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include "sof_sdw_common.h" +#include "../../codecs/hdac_hdmi.h" +#include "hda_dsp_common.h" + +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC) +static struct snd_soc_jack hdmi[MAX_HDMI_NUM]; + +struct hdmi_pcm { + struct list_head head; + struct snd_soc_dai *codec_dai; + int device; +}; + +int sof_sdw_hdmi_init(struct snd_soc_pcm_runtime *rtd) +{ + struct mc_private *ctx = snd_soc_card_get_drvdata(rtd->card); + struct snd_soc_dai *dai = rtd->codec_dai; + struct hdmi_pcm *pcm; + + pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL); + if (!pcm) + return -ENOMEM; + + /* dai_link id is 1:1 mapped to the PCM device */ + pcm->device = rtd->dai_link->id; + pcm->codec_dai = dai; + + list_add_tail(&pcm->head, &ctx->hdmi_pcm_list); + + return 0; +} + +#define NAME_SIZE 32 +int sof_sdw_hdmi_card_late_probe(struct snd_soc_card *card) +{ + struct mc_private *ctx = snd_soc_card_get_drvdata(card); + struct hdmi_pcm *pcm; + struct snd_soc_component *component = NULL; + int err, i = 0; + char jack_name[NAME_SIZE]; + + pcm = list_first_entry(&ctx->hdmi_pcm_list, struct hdmi_pcm, + head); + component = pcm->codec_dai->component; + + if (ctx->common_hdmi_codec_drv) + return hda_dsp_hdmi_build_controls(card, component); + + list_for_each_entry(pcm, &ctx->hdmi_pcm_list, head) { + component = pcm->codec_dai->component; + snprintf(jack_name, sizeof(jack_name), + "HDMI/DP, pcm=%d Jack", pcm->device); + err = snd_soc_card_jack_new(card, jack_name, + SND_JACK_AVOUT, &hdmi[i], + NULL, 0); + + if (err) + return err; + + err = snd_jack_add_new_kctl(hdmi[i].jack, + jack_name, SND_JACK_AVOUT); + if (err) + dev_warn(component->dev, "failed creating Jack kctl\n"); + + err = hdac_hdmi_jack_init(pcm->codec_dai, pcm->device, + &hdmi[i]); + if (err < 0) + return err; + + i++; + } + + if (!component) + return -EINVAL; + + return hdac_hdmi_jack_port_init(component, &card->dapm); +} +#else +int hdmi_card_late_probe(struct snd_soc_card *card) +{ + return 0; +} +#endif diff --git a/sound/soc/intel/boards/sof_sdw_rt1308.c b/sound/soc/intel/boards/sof_sdw_rt1308.c new file mode 100644 index 000000000000..321768e54d08 --- /dev/null +++ b/sound/soc/intel/boards/sof_sdw_rt1308.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Intel Corporation + +/* + * sof_sdw_rt1308 - Helpers to handle RT1308 from generic machine driver + */ + +#include +#include +#include +#include +#include "sof_sdw_common.h" +#include "../../codecs/rt1308.h" + +static const struct snd_soc_dapm_widget rt1308_widgets[] = { + SND_SOC_DAPM_SPK("Speaker", NULL), +}; + +/* + * dapm routes for rt1308 will be registered dynamically according + * to the number of rt1308 used. The first two entries will be registered + * for one codec case, and the last two entries are also registered + * if two 1308s are used. + */ +static const struct snd_soc_dapm_route rt1308_map[] = { + { "Speaker", NULL, "rt1308-1 SPOL" }, + { "Speaker", NULL, "rt1308-1 SPOR" }, + { "Speaker", NULL, "rt1308-2 SPOL" }, + { "Speaker", NULL, "rt1308-2 SPOR" }, +}; + +static const struct snd_kcontrol_new rt1308_controls[] = { + SOC_DAPM_PIN_SWITCH("Speaker"), +}; + +static int first_spk_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + int ret; + + card->components = devm_kasprintf(card->dev, GFP_KERNEL, + "%s spk:rt1308", + card->components); + if (!card->components) + return -ENOMEM; + + ret = snd_soc_add_card_controls(card, rt1308_controls, + ARRAY_SIZE(rt1308_controls)); + if (ret) { + dev_err(card->dev, "rt1308 controls addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_dapm_new_controls(&card->dapm, rt1308_widgets, + ARRAY_SIZE(rt1308_widgets)); + if (ret) { + dev_err(card->dev, "rt1308 widgets addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, rt1308_map, 2); + if (ret) + dev_err(rtd->dev, "failed to add first SPK map: %d\n", ret); + + return ret; +} + +static int second_spk_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + int ret; + + ret = snd_soc_dapm_add_routes(&card->dapm, rt1308_map + 2, 2); + if (ret) + dev_err(rtd->dev, "failed to add second SPK map: %d\n", ret); + + return ret; +} + +static int all_spk_init(struct snd_soc_pcm_runtime *rtd) +{ + int ret; + + ret = first_spk_init(rtd); + if (ret) + return ret; + + return second_spk_init(rtd); +} + +static int rt1308_i2s_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_card *card = rtd->card; + struct snd_soc_dai *codec_dai = rtd->codec_dai; + int clk_id, clk_freq, pll_out; + int err; + + clk_id = RT1308_PLL_S_MCLK; + clk_freq = 38400000; + + pll_out = params_rate(params) * 512; + + /* Set rt1308 pll */ + err = snd_soc_dai_set_pll(codec_dai, 0, clk_id, clk_freq, pll_out); + if (err < 0) { + dev_err(card->dev, "Failed to set RT1308 PLL: %d\n", err); + return err; + } + + /* Set rt1308 sysclk */ + err = snd_soc_dai_set_sysclk(codec_dai, RT1308_FS_SYS_S_PLL, pll_out, + SND_SOC_CLOCK_IN); + if (err < 0) { + dev_err(card->dev, "Failed to set RT1308 SYSCLK: %d\n", err); + return err; + } + + return 0; +} + +/* machine stream operations */ +struct snd_soc_ops sof_sdw_rt1308_i2s_ops = { + .hw_params = rt1308_i2s_hw_params, +}; + +int sof_sdw_rt1308_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback) +{ + info->amp_num++; + if (info->amp_num == 1) + dai_links->init = first_spk_init; + + if (info->amp_num == 2) { + /* + * if two 1308s are in one dai link, the init function + * in this dai link will be first set for the first speaker, + * and it should be reset to initialize all speakers when + * the second speaker is found. + */ + if (dai_links->init) + dai_links->init = all_spk_init; + else + dai_links->init = second_spk_init; + } + + return 0; +} diff --git a/sound/soc/intel/boards/sof_sdw_rt5682.c b/sound/soc/intel/boards/sof_sdw_rt5682.c new file mode 100644 index 000000000000..5aa6211a1ed9 --- /dev/null +++ b/sound/soc/intel/boards/sof_sdw_rt5682.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Intel Corporation + +/* + * sof_sdw_rt5682 - Helpers to handle RT5682 from generic machine driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "sof_sdw_common.h" + +static const struct snd_soc_dapm_widget rt5682_widgets[] = { + SND_SOC_DAPM_HP("Headphone", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), +}; + +static const struct snd_soc_dapm_route rt5682_map[] = { + /*Headphones*/ + { "Headphone", NULL, "rt5682 HPOL" }, + { "Headphone", NULL, "rt5682 HPOR" }, + { "rt5682 IN1P", NULL, "Headset Mic" }, +}; + +static const struct snd_kcontrol_new rt5682_controls[] = { + SOC_DAPM_PIN_SWITCH("Headphone"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), +}; + +static struct snd_soc_jack_pin rt5682_jack_pins[] = { + { + .pin = "Headphone", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, +}; + +static int rt5682_rtd_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct mc_private *ctx = snd_soc_card_get_drvdata(card); + struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_jack *jack; + int ret; + + card->components = devm_kasprintf(card->dev, GFP_KERNEL, + "%s hs:rt5682", + card->components); + if (!card->components) + return -ENOMEM; + + ret = snd_soc_add_card_controls(card, rt5682_controls, + ARRAY_SIZE(rt5682_controls)); + if (ret) { + dev_err(card->dev, "rt5682 control addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_dapm_new_controls(&card->dapm, rt5682_widgets, + ARRAY_SIZE(rt5682_widgets)); + if (ret) { + dev_err(card->dev, "rt5682 widgets addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, rt5682_map, + ARRAY_SIZE(rt5682_map)); + + if (ret) { + dev_err(card->dev, "rt5682 map addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", + SND_JACK_HEADSET | SND_JACK_BTN_0 | + SND_JACK_BTN_1 | SND_JACK_BTN_2 | + SND_JACK_BTN_3, + &ctx->sdw_headset, + rt5682_jack_pins, + ARRAY_SIZE(rt5682_jack_pins)); + if (ret) { + dev_err(rtd->card->dev, "Headset Jack creation failed: %d\n", + ret); + return ret; + } + + jack = &ctx->sdw_headset; + + snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_PLAYPAUSE); + snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_VOICECOMMAND); + snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEUP); + snd_jack_set_key(jack->jack, SND_JACK_BTN_3, KEY_VOLUMEDOWN); + + ret = snd_soc_component_set_jack(component, jack, NULL); + + if (ret) + dev_err(rtd->card->dev, "Headset Jack call-back failed: %d\n", + ret); + + return ret; +} + +int sof_sdw_rt5682_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback) +{ + /* + * headset should be initialized once. + * Do it with dai link for playback. + */ + if (!playback) + return 0; + + dai_links->init = rt5682_rtd_init; + + return 0; +} diff --git a/sound/soc/intel/boards/sof_sdw_rt700.c b/sound/soc/intel/boards/sof_sdw_rt700.c new file mode 100644 index 000000000000..2ee4e6910d7f --- /dev/null +++ b/sound/soc/intel/boards/sof_sdw_rt700.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Intel Corporation + +/* + * sof_sdw_rt700 - Helpers to handle RT700 from generic machine driver + */ + +#include +#include +#include +#include +#include +#include +#include "sof_sdw_common.h" + +static const struct snd_soc_dapm_widget rt700_widgets[] = { + SND_SOC_DAPM_HP("Headphones", NULL), + SND_SOC_DAPM_MIC("AMIC", NULL), + SND_SOC_DAPM_SPK("Speaker", NULL), +}; + +static const struct snd_soc_dapm_route rt700_map[] = { + /* Headphones */ + { "Headphones", NULL, "HP" }, + { "Speaker", NULL, "SPK" }, + { "MIC2", NULL, "AMIC" }, +}; + +static const struct snd_kcontrol_new rt700_controls[] = { + SOC_DAPM_PIN_SWITCH("Headphones"), + SOC_DAPM_PIN_SWITCH("AMIC"), + SOC_DAPM_PIN_SWITCH("Speaker"), +}; + +static struct snd_soc_jack_pin rt700_jack_pins[] = { + { + .pin = "Headphones", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "AMIC", + .mask = SND_JACK_MICROPHONE, + }, +}; + +static int rt700_rtd_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct mc_private *ctx = snd_soc_card_get_drvdata(card); + struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_jack *jack; + int ret; + + card->components = devm_kasprintf(card->dev, GFP_KERNEL, + "%s hs:rt700", + card->components); + if (!card->components) + return -ENOMEM; + + ret = snd_soc_add_card_controls(card, rt700_controls, + ARRAY_SIZE(rt700_controls)); + if (ret) { + dev_err(card->dev, "rt700 controls addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_dapm_new_controls(&card->dapm, rt700_widgets, + ARRAY_SIZE(rt700_widgets)); + if (ret) { + dev_err(card->dev, "rt700 widgets addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, rt700_map, + ARRAY_SIZE(rt700_map)); + + if (ret) { + dev_err(card->dev, "rt700 map addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", + SND_JACK_HEADSET | SND_JACK_BTN_0 | + SND_JACK_BTN_1 | SND_JACK_BTN_2 | + SND_JACK_BTN_3, + &ctx->sdw_headset, + rt700_jack_pins, + ARRAY_SIZE(rt700_jack_pins)); + if (ret) { + dev_err(rtd->card->dev, "Headset Jack creation failed: %d\n", + ret); + return ret; + } + + jack = &ctx->sdw_headset; + + snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_VOLUMEUP); + snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_PLAYPAUSE); + snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN); + snd_jack_set_key(jack->jack, SND_JACK_BTN_3, KEY_VOICECOMMAND); + + ret = snd_soc_component_set_jack(component, jack, NULL); + if (ret) + dev_err(rtd->card->dev, "Headset Jack call-back failed: %d\n", + ret); + + return ret; +} + +int sof_sdw_rt700_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback) +{ + /* + * headset should be initialized once. + * Do it with dai link for playback. + */ + if (!playback) + return 0; + + dai_links->init = rt700_rtd_init; + + return 0; +} diff --git a/sound/soc/intel/boards/sof_sdw_rt711.c b/sound/soc/intel/boards/sof_sdw_rt711.c new file mode 100644 index 000000000000..2a4917e3d561 --- /dev/null +++ b/sound/soc/intel/boards/sof_sdw_rt711.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Intel Corporation + +/* + * sof_sdw_rt711 - Helpers to handle RT711 from generic machine driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "sof_sdw_common.h" + +/* + * Note this MUST be called before snd_soc_register_card(), so that the props + * are in place before the codec component driver's probe function parses them. + */ +static int rt711_add_codec_device_props(const char *sdw_dev_name) +{ + struct property_entry props[MAX_NO_PROPS] = {}; + struct device *sdw_dev; + int ret; + + sdw_dev = bus_find_device_by_name(&sdw_bus_type, NULL, sdw_dev_name); + if (!sdw_dev) + return -EPROBE_DEFER; + + if (SOF_RT711_JDSRC(sof_sdw_quirk)) { + props[0] = PROPERTY_ENTRY_U32("realtek,jd-src", + SOF_RT711_JDSRC(sof_sdw_quirk)); + } + + ret = device_add_properties(sdw_dev, props); + put_device(sdw_dev); + + return ret; +} + +static const struct snd_soc_dapm_widget rt711_widgets[] = { + SND_SOC_DAPM_HP("Headphone", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), +}; + +static const struct snd_soc_dapm_route rt711_map[] = { + /* Headphones */ + { "Headphone", NULL, "rt711 HP" }, + { "rt711 MIC2", NULL, "Headset Mic" }, +}; + +static const struct snd_kcontrol_new rt711_controls[] = { + SOC_DAPM_PIN_SWITCH("Headphone"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), +}; + +static struct snd_soc_jack_pin rt711_jack_pins[] = { + { + .pin = "Headphone", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, +}; + +static int rt711_rtd_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct mc_private *ctx = snd_soc_card_get_drvdata(card); + struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_jack *jack; + int ret; + + card->components = devm_kasprintf(card->dev, GFP_KERNEL, + "%s hs:rt711", + card->components); + if (!card->components) + return -ENOMEM; + + ret = snd_soc_add_card_controls(card, rt711_controls, + ARRAY_SIZE(rt711_controls)); + if (ret) { + dev_err(card->dev, "rt711 controls addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_dapm_new_controls(&card->dapm, rt711_widgets, + ARRAY_SIZE(rt711_widgets)); + if (ret) { + dev_err(card->dev, "rt711 widgets addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, rt711_map, + ARRAY_SIZE(rt711_map)); + + if (ret) { + dev_err(card->dev, "rt711 map addition failed: %d\n", ret); + return ret; + } + + ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", + SND_JACK_HEADSET | SND_JACK_BTN_0 | + SND_JACK_BTN_1 | SND_JACK_BTN_2 | + SND_JACK_BTN_3, + &ctx->sdw_headset, + rt711_jack_pins, + ARRAY_SIZE(rt711_jack_pins)); + if (ret) { + dev_err(rtd->card->dev, "Headset Jack creation failed: %d\n", + ret); + return ret; + } + + jack = &ctx->sdw_headset; + + snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_VOLUMEUP); + snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_PLAYPAUSE); + snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN); + snd_jack_set_key(jack->jack, SND_JACK_BTN_3, KEY_VOICECOMMAND); + + ret = snd_soc_component_set_jack(component, jack, NULL); + + if (ret) + dev_err(rtd->card->dev, "Headset Jack call-back failed: %d\n", + ret); + + return ret; +} + +int sof_sdw_rt711_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback) +{ + int ret; + + /* + * headset should be initialized once. + * Do it with dai link for playback. + */ + if (!playback) + return 0; + + ret = rt711_add_codec_device_props("sdw:0:25d:711:0"); + if (ret < 0) + return ret; + + dai_links->init = rt711_rtd_init; + + return 0; +} diff --git a/sound/soc/intel/boards/sof_sdw_rt715.c b/sound/soc/intel/boards/sof_sdw_rt715.c new file mode 100644 index 000000000000..321e1cbc03ed --- /dev/null +++ b/sound/soc/intel/boards/sof_sdw_rt715.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Intel Corporation + +/* + * sof_sdw_rt715 - Helpers to handle RT715 from generic machine driver + */ + +#include +#include +#include +#include +#include "sof_sdw_common.h" + +static int rt715_rtd_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + + card->components = devm_kasprintf(card->dev, GFP_KERNEL, + "%s mic:rt715", + card->components); + if (!card->components) + return -ENOMEM; + + return 0; +} + +int sof_sdw_rt715_init(const struct snd_soc_acpi_link_adr *link, + struct snd_soc_dai_link *dai_links, + struct sof_sdw_codec_info *info, + bool playback) +{ + /* + * DAI ID is fixed at SDW_DMIC_DAI_ID for 715 to + * keep sdw DMIC and HDMI setting static in UCM + */ + if (sof_sdw_quirk & SOF_RT715_DAI_ID_FIX) + dai_links->id = SDW_DMIC_DAI_ID; + + dai_links->init = rt715_rtd_init; + + return 0; +} diff --git a/sound/soc/intel/common/soc-acpi-intel-bxt-match.c b/sound/soc/intel/common/soc-acpi-intel-bxt-match.c index 4a5adae1d785..f5092bc48364 100644 --- a/sound/soc/intel/common/soc-acpi-intel-bxt-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-bxt-match.c @@ -65,7 +65,7 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_bxt_machines[] = { }, { .id = "104C5122", - .drv_name = "bxt-pcm512x", + .drv_name = "sof_pcm512x", .sof_fw_filename = "sof-apl.ri", .sof_tplg_filename = "sof-apl-pcm512x.tplg", }, diff --git a/sound/soc/intel/common/soc-acpi-intel-cht-match.c b/sound/soc/intel/common/soc-acpi-intel-cht-match.c index d0fb43c2b9f6..2752dc955733 100644 --- a/sound/soc/intel/common/soc-acpi-intel-cht-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-cht-match.c @@ -174,6 +174,13 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_cherrytrail_machines[] = { .sof_fw_filename = "sof-cht.ri", .sof_tplg_filename = "sof-cht-cx2072x.tplg", }, + { + .id = "104C5122", + .drv_name = "sof_pcm512x", + .sof_fw_filename = "sof-cht.ri", + .sof_tplg_filename = "sof-cht-src-50khz-pcm512x.tplg", + }, + #if IS_ENABLED(CONFIG_SND_SOC_INTEL_BYT_CHT_NOCODEC_MACH) /* * This is always last in the table so that it is selected only when diff --git a/sound/soc/intel/common/soc-acpi-intel-cml-match.c b/sound/soc/intel/common/soc-acpi-intel-cml-match.c index f55634c4c2e8..bcedec6c6117 100644 --- a/sound/soc/intel/common/soc-acpi-intel-cml-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-cml-match.c @@ -59,42 +59,112 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_cml_machines[] = { }; EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_cml_machines); -static const u64 rt711_0_adr[] = { - 0x000010025D071100 +static const struct snd_soc_acpi_endpoint single_endpoint = { + .num = 0, + .aggregated = 0, + .group_position = 0, + .group_id = 0, }; -static const u64 rt1308_1_adr[] = { - 0x000110025D130800 +static const struct snd_soc_acpi_endpoint spk_l_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 0, + .group_id = 1, }; -static const u64 rt1308_2_adr[] = { - 0x000210025D130800 +static const struct snd_soc_acpi_endpoint spk_r_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 1, + .group_id = 1, }; -static const u64 rt715_3_adr[] = { - 0x000310025D071500 +static const struct snd_soc_acpi_adr_device rt700_1_adr[] = { + { + .adr = 0x000110025D070000, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } +}; + +static const struct snd_soc_acpi_link_adr cml_rvp[] = { + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(rt700_1_adr), + .adr_d = rt700_1_adr, + }, + {} +}; + +static const struct snd_soc_acpi_adr_device rt711_0_adr[] = { + { + .adr = 0x000010025D071100, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } +}; + +static const struct snd_soc_acpi_adr_device rt1308_1_adr[] = { + { + .adr = 0x000110025D130800, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } +}; + +static const struct snd_soc_acpi_adr_device rt1308_2_adr[] = { + { + .adr = 0x000210025D130800, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } +}; + +static const struct snd_soc_acpi_adr_device rt1308_1_group1_adr[] = { + { + .adr = 0x000110025D130800, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + } +}; + +static const struct snd_soc_acpi_adr_device rt1308_2_group1_adr[] = { + { + .adr = 0x000210025D130800, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + } +}; + +static const struct snd_soc_acpi_adr_device rt715_3_adr[] = { + { + .adr = 0x000310025D071500, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } }; static const struct snd_soc_acpi_link_adr cml_3_in_1_default[] = { { .mask = BIT(0), .num_adr = ARRAY_SIZE(rt711_0_adr), - .adr = rt711_0_adr, + .adr_d = rt711_0_adr, }, { .mask = BIT(1), - .num_adr = ARRAY_SIZE(rt1308_1_adr), - .adr = rt1308_1_adr, + .num_adr = ARRAY_SIZE(rt1308_1_group1_adr), + .adr_d = rt1308_1_group1_adr, }, { .mask = BIT(2), - .num_adr = ARRAY_SIZE(rt1308_2_adr), - .adr = rt1308_2_adr, + .num_adr = ARRAY_SIZE(rt1308_2_group1_adr), + .adr_d = rt1308_2_group1_adr, }, { .mask = BIT(3), .num_adr = ARRAY_SIZE(rt715_3_adr), - .adr = rt715_3_adr, + .adr_d = rt715_3_adr, }, {} }; @@ -103,17 +173,17 @@ static const struct snd_soc_acpi_link_adr cml_3_in_1_mono_amp[] = { { .mask = BIT(0), .num_adr = ARRAY_SIZE(rt711_0_adr), - .adr = rt711_0_adr, + .adr_d = rt711_0_adr, }, { .mask = BIT(1), .num_adr = ARRAY_SIZE(rt1308_1_adr), - .adr = rt1308_1_adr, + .adr_d = rt1308_1_adr, }, { .mask = BIT(3), .num_adr = ARRAY_SIZE(rt715_3_adr), - .adr = rt715_3_adr, + .adr_d = rt715_3_adr, }, {} }; @@ -122,7 +192,7 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_cml_sdw_machines[] = { { .link_mask = 0xF, /* 4 active links required */ .links = cml_3_in_1_default, - .drv_name = "sdw_rt711_rt1308_rt715", + .drv_name = "sof_sdw", .sof_fw_filename = "sof-cml.ri", .sof_tplg_filename = "sof-cml-rt711-rt1308-rt715.tplg", }, @@ -134,13 +204,14 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_cml_sdw_machines[] = { */ .link_mask = 0xF, .links = cml_3_in_1_mono_amp, - .drv_name = "sdw_rt711_rt1308_rt715", + .drv_name = "sof_sdw", .sof_fw_filename = "sof-cml.ri", .sof_tplg_filename = "sof-cml-rt711-rt1308-mono-rt715.tplg", }, { .link_mask = 0x2, /* RT700 connected on Link1 */ - .drv_name = "sdw_rt700", + .links = cml_rvp, + .drv_name = "sof_sdw", .sof_fw_filename = "sof-cml.ri", .sof_tplg_filename = "sof-cml-rt700.tplg", }, diff --git a/sound/soc/intel/common/soc-acpi-intel-icl-match.c b/sound/soc/intel/common/soc-acpi-intel-icl-match.c index 752733013d54..ef8500349f2f 100644 --- a/sound/soc/intel/common/soc-acpi-intel-icl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-icl-match.c @@ -33,55 +33,112 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_icl_machines[] = { }; EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_icl_machines); -static const u64 rt700_0_adr[] = { - 0x000010025D070000 +static const struct snd_soc_acpi_endpoint single_endpoint = { + .num = 0, + .aggregated = 0, + .group_position = 0, + .group_id = 0, +}; + +static const struct snd_soc_acpi_endpoint spk_l_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 0, + .group_id = 1, +}; + +static const struct snd_soc_acpi_endpoint spk_r_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 1, + .group_id = 1, +}; + +static const struct snd_soc_acpi_adr_device rt700_0_adr[] = { + { + .adr = 0x000010025D070000, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } }; static const struct snd_soc_acpi_link_adr icl_rvp[] = { { .mask = BIT(0), .num_adr = ARRAY_SIZE(rt700_0_adr), - .adr = rt700_0_adr, + .adr_d = rt700_0_adr, }, {} }; -static const u64 rt711_0_adr[] = { - 0x000010025D071100 +static const struct snd_soc_acpi_adr_device rt711_0_adr[] = { + { + .adr = 0x000010025D071100, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } }; -static const u64 rt1308_1_adr[] = { - 0x000110025D130800 +static const struct snd_soc_acpi_adr_device rt1308_1_adr[] = { + { + .adr = 0x000110025D130800, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } }; -static const u64 rt1308_2_adr[] = { - 0x000210025D130800 +static const struct snd_soc_acpi_adr_device rt1308_2_adr[] = { + { + .adr = 0x000210025D130800, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } }; -static const u64 rt715_3_adr[] = { - 0x000310025D071500 +static const struct snd_soc_acpi_adr_device rt1308_1_group1_adr[] = { + { + .adr = 0x000110025D130800, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + } +}; + +static const struct snd_soc_acpi_adr_device rt1308_2_group1_adr[] = { + { + .adr = 0x000210025D130800, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + } +}; + +static const struct snd_soc_acpi_adr_device rt715_3_adr[] = { + { + .adr = 0x000310025D071500, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } }; static const struct snd_soc_acpi_link_adr icl_3_in_1_default[] = { { .mask = BIT(0), .num_adr = ARRAY_SIZE(rt711_0_adr), - .adr = rt711_0_adr, + .adr_d = rt711_0_adr, }, { .mask = BIT(1), - .num_adr = ARRAY_SIZE(rt1308_1_adr), - .adr = rt1308_1_adr, + .num_adr = ARRAY_SIZE(rt1308_1_group1_adr), + .adr_d = rt1308_1_group1_adr, }, { .mask = BIT(2), - .num_adr = ARRAY_SIZE(rt1308_2_adr), - .adr = rt1308_2_adr, + .num_adr = ARRAY_SIZE(rt1308_2_group1_adr), + .adr_d = rt1308_2_group1_adr, }, { .mask = BIT(3), .num_adr = ARRAY_SIZE(rt715_3_adr), - .adr = rt715_3_adr, + .adr_d = rt715_3_adr, }, {} }; @@ -90,17 +147,17 @@ static const struct snd_soc_acpi_link_adr icl_3_in_1_mono_amp[] = { { .mask = BIT(0), .num_adr = ARRAY_SIZE(rt711_0_adr), - .adr = rt711_0_adr, + .adr_d = rt711_0_adr, }, { .mask = BIT(1), .num_adr = ARRAY_SIZE(rt1308_1_adr), - .adr = rt1308_1_adr, + .adr_d = rt1308_1_adr, }, { .mask = BIT(3), .num_adr = ARRAY_SIZE(rt715_3_adr), - .adr = rt715_3_adr, + .adr_d = rt715_3_adr, }, {} }; @@ -109,21 +166,21 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_icl_sdw_machines[] = { { .link_mask = 0xF, /* 4 active links required */ .links = icl_3_in_1_default, - .drv_name = "sdw_rt711_rt1308_rt715", + .drv_name = "sof_sdw", .sof_fw_filename = "sof-icl.ri", .sof_tplg_filename = "sof-icl-rt711-rt1308-rt715.tplg", }, { .link_mask = 0xB, /* 3 active links required */ .links = icl_3_in_1_mono_amp, - .drv_name = "sdw_rt711_rt1308_rt715", + .drv_name = "sof_sdw", .sof_fw_filename = "sof-icl.ri", .sof_tplg_filename = "sof-icl-rt711-rt1308-rt715-mono.tplg", }, { .link_mask = 0x1, /* rt700 connected on link0 */ .links = icl_rvp, - .drv_name = "sdw_rt700", + .drv_name = "sof_sdw", .sof_fw_filename = "sof-icl.ri", .sof_tplg_filename = "sof-icl-rt700.tplg", }, diff --git a/sound/soc/intel/common/soc-acpi-intel-jsl-match.c b/sound/soc/intel/common/soc-acpi-intel-jsl-match.c index ed2b125f6a11..4388a32718d8 100644 --- a/sound/soc/intel/common/soc-acpi-intel-jsl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-jsl-match.c @@ -2,20 +2,50 @@ /* * soc-apci-intel-jsl-match.c - tables and support for JSL ACPI enumeration. * - * Copyright (c) 2019, Intel Corporation. + * Copyright (c) 2019-2020, Intel Corporation. * */ #include #include +static struct snd_soc_acpi_codecs jsl_7219_98373_codecs = { + .num_codecs = 1, + .codecs = {"MX98373"} +}; + +static struct snd_soc_acpi_codecs rt1015_spk = { + .num_codecs = 1, + .codecs = {"10EC1015"} +}; + +/* + * When adding new entry to the snd_soc_acpi_intel_jsl_machines array, + * use .quirk_data member to distinguish different machine driver, + * and keep ACPI .id field unchanged for the common codec. + */ struct snd_soc_acpi_mach snd_soc_acpi_intel_jsl_machines[] = { { .id = "DLGS7219", .drv_name = "sof_da7219_max98373", - .machine_quirk = snd_soc_acpi_codec_list, .sof_fw_filename = "sof-jsl.ri", .sof_tplg_filename = "sof-jsl-da7219.tplg", + .machine_quirk = snd_soc_acpi_codec_list, + .quirk_data = &jsl_7219_98373_codecs, + }, + { + .id = "DLGS7219", + .drv_name = "sof_da7219_max98360a", + .sof_fw_filename = "sof-jsl.ri", + .sof_tplg_filename = "sof-jsl-da7219-mx98360a.tplg", + }, + { + .id = "10EC5682", + .drv_name = "jsl_rt5682_rt1015", + .sof_fw_filename = "sof-jsl.ri", + .machine_quirk = snd_soc_acpi_codec_list, + .quirk_data = &rt1015_spk, + .sof_tplg_filename = "sof-jsl-rt5682-rt1015.tplg", }, {}, }; diff --git a/sound/soc/intel/common/soc-acpi-intel-tgl-match.c b/sound/soc/intel/common/soc-acpi-intel-tgl-match.c index 5984dd151f3e..449d9d2286ae 100644 --- a/sound/soc/intel/common/soc-acpi-intel-tgl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-tgl-match.c @@ -14,20 +14,61 @@ static struct snd_soc_acpi_codecs tgl_codecs = { .codecs = {"MX98357A"} }; -static const u64 rt711_0_adr[] = { - 0x000010025D071100 +static const struct snd_soc_acpi_endpoint single_endpoint = { + .num = 0, + .aggregated = 0, + .group_position = 0, + .group_id = 0, }; -static const u64 rt1308_1_adr[] = { - 0x000120025D130800, - 0x000122025D130800 +static const struct snd_soc_acpi_endpoint spk_l_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 0, + .group_id = 1, +}; + +static const struct snd_soc_acpi_endpoint spk_r_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 1, + .group_id = 1, +}; + +static const struct snd_soc_acpi_adr_device rt711_0_adr[] = { + { + .adr = 0x000010025D071100, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } +}; + +static const struct snd_soc_acpi_adr_device rt1308_1_adr[] = { + { + .adr = 0x000120025D130800, + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + }, + { + .adr = 0x000122025D130800, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + } +}; + +static const struct snd_soc_acpi_adr_device rt5682_0_adr[] = { + { + .adr = 0x000021025D568200, + .num_endpoints = 1, + .endpoints = &single_endpoint, + } }; static const struct snd_soc_acpi_link_adr tgl_i2s_rt1308[] = { { .mask = BIT(0), .num_adr = ARRAY_SIZE(rt711_0_adr), - .adr = rt711_0_adr, + .adr_d = rt711_0_adr, }, {} }; @@ -36,24 +77,38 @@ static const struct snd_soc_acpi_link_adr tgl_rvp[] = { { .mask = BIT(0), .num_adr = ARRAY_SIZE(rt711_0_adr), - .adr = rt711_0_adr, + .adr_d = rt711_0_adr, }, { .mask = BIT(1), .num_adr = ARRAY_SIZE(rt1308_1_adr), - .adr = rt1308_1_adr, + .adr_d = rt1308_1_adr, }, {} }; +static const struct snd_soc_acpi_link_adr tgl_chromebook_base[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(rt5682_0_adr), + .adr_d = rt5682_0_adr, + }, + {} +}; + +static struct snd_soc_acpi_codecs tgl_max98373_amp = { + .num_codecs = 1, + .codecs = {"MX98373"} +}; + struct snd_soc_acpi_mach snd_soc_acpi_intel_tgl_machines[] = { { .id = "10EC1308", - .drv_name = "rt711_rt1308", + .drv_name = "sof_sdw", .link_mask = 0x1, /* RT711 on SoundWire link0 */ .links = tgl_i2s_rt1308, .sof_fw_filename = "sof-tgl.ri", - .sof_tplg_filename = "sof-tgl-rt711-rt1308.tplg", + .sof_tplg_filename = "sof-tgl-rt711-i2s-rt1308.tplg", }, { .id = "10EC5682", @@ -63,6 +118,14 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_tgl_machines[] = { .sof_fw_filename = "sof-tgl.ri", .sof_tplg_filename = "sof-tgl-max98357a-rt5682.tplg", }, + { + .id = "10EC5682", + .drv_name = "tgl_max98373_rt5682", + .machine_quirk = snd_soc_acpi_codec_list, + .quirk_data = &tgl_max98373_amp, + .sof_fw_filename = "sof-tgl.ri", + .sof_tplg_filename = "sof-tgl-max98373-rt5682.tplg", + }, {}, }; EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_tgl_machines); @@ -72,10 +135,17 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_tgl_sdw_machines[] = { { .link_mask = 0x3, /* rt711 on link 0 and 2 rt1308s on link 1 */ .links = tgl_rvp, - .drv_name = "sdw_rt711_rt1308_rt715", + .drv_name = "sof_sdw", .sof_fw_filename = "sof-tgl.ri", .sof_tplg_filename = "sof-tgl-rt711-rt1308.tplg", }, + { + .link_mask = 0x1, /* this will only enable rt5682 for now */ + .links = tgl_chromebook_base, + .drv_name = "sof_sdw", + .sof_fw_filename = "sof-tgl.ri", + .sof_tplg_filename = "sof-tgl-rt5682.tplg", + }, {}, }; EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_tgl_sdw_machines); diff --git a/sound/soc/intel/haswell/sst-haswell-pcm.c b/sound/soc/intel/haswell/sst-haswell-pcm.c index 033d7c05d7fb..c183f8e94ee4 100644 --- a/sound/soc/intel/haswell/sst-haswell-pcm.c +++ b/sound/soc/intel/haswell/sst-haswell-pcm.c @@ -476,7 +476,7 @@ static int hsw_pcm_hw_params(struct snd_soc_component *component, u8 channels; int ret, dai; - dai = mod_map[rtd->cpu_dai->id].dai_id; + dai = mod_map[asoc_rtd_to_cpu(rtd, 0)->id].dai_id; pcm_data = &pdata->pcm[dai][substream->stream]; /* check if we are being called a subsequent time */ @@ -494,7 +494,7 @@ static int hsw_pcm_hw_params(struct snd_soc_component *component, } pcm_data->allocated = false; - pcm_data->stream = sst_hsw_stream_new(hsw, rtd->cpu_dai->id, + pcm_data->stream = sst_hsw_stream_new(hsw, asoc_rtd_to_cpu(rtd, 0)->id, hsw_notify_pointer, pcm_data); if (pcm_data->stream == NULL) { dev_err(rtd->dev, "error: failed to create stream\n"); @@ -509,7 +509,7 @@ static int hsw_pcm_hw_params(struct snd_soc_component *component, path_id = SST_HSW_STREAM_PATH_SSP0_IN; /* DSP stream type depends on DAI ID */ - switch (rtd->cpu_dai->id) { + switch (asoc_rtd_to_cpu(rtd, 0)->id) { case 0: if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { stream_type = SST_HSW_STREAM_TYPE_SYSTEM; @@ -533,7 +533,7 @@ static int hsw_pcm_hw_params(struct snd_soc_component *component, break; default: dev_err(rtd->dev, "error: invalid DAI ID %d\n", - rtd->cpu_dai->id); + asoc_rtd_to_cpu(rtd, 0)->id); return -EINVAL; } @@ -595,7 +595,7 @@ static int hsw_pcm_hw_params(struct snd_soc_component *component, dmab = snd_pcm_get_dma_buf(substream); ret = create_adsp_page_table(substream, pdata, rtd, runtime->dma_area, - runtime->dma_bytes, rtd->cpu_dai->id); + runtime->dma_bytes, asoc_rtd_to_cpu(rtd, 0)->id); if (ret < 0) return ret; @@ -608,7 +608,7 @@ static int hsw_pcm_hw_params(struct snd_soc_component *component, pages = runtime->dma_bytes / PAGE_SIZE; ret = sst_hsw_stream_buffer(hsw, pcm_data->stream, - pdata->dmab[rtd->cpu_dai->id][substream->stream].addr, + pdata->dmab[asoc_rtd_to_cpu(rtd, 0)->id][substream->stream].addr, pages, runtime->dma_bytes, 0, snd_sgbuf_get_addr(dmab, 0) >> PAGE_SHIFT); if (ret < 0) { @@ -661,7 +661,7 @@ static int hsw_pcm_trigger(struct snd_soc_component *component, snd_pcm_uframes_t pos; int dai; - dai = mod_map[rtd->cpu_dai->id].dai_id; + dai = mod_map[asoc_rtd_to_cpu(rtd, 0)->id].dai_id; pcm_data = &pdata->pcm[dai][substream->stream]; sst_stream = pcm_data->stream; @@ -770,7 +770,7 @@ static snd_pcm_uframes_t hsw_pcm_pointer(struct snd_soc_component *component, u32 position; int dai; - dai = mod_map[rtd->cpu_dai->id].dai_id; + dai = mod_map[asoc_rtd_to_cpu(rtd, 0)->id].dai_id; pcm_data = &pdata->pcm[dai][substream->stream]; position = sst_hsw_get_dsp_position(hsw, pcm_data->stream); @@ -791,7 +791,7 @@ static int hsw_pcm_open(struct snd_soc_component *component, struct sst_hsw *hsw = pdata->hsw; int dai; - dai = mod_map[rtd->cpu_dai->id].dai_id; + dai = mod_map[asoc_rtd_to_cpu(rtd, 0)->id].dai_id; pcm_data = &pdata->pcm[dai][substream->stream]; mutex_lock(&pcm_data->mutex); @@ -801,7 +801,7 @@ static int hsw_pcm_open(struct snd_soc_component *component, snd_soc_set_runtime_hwparams(substream, &hsw_pcm_hardware); - pcm_data->stream = sst_hsw_stream_new(hsw, rtd->cpu_dai->id, + pcm_data->stream = sst_hsw_stream_new(hsw, asoc_rtd_to_cpu(rtd, 0)->id, hsw_notify_pointer, pcm_data); if (pcm_data->stream == NULL) { dev_err(rtd->dev, "error: failed to create stream\n"); @@ -824,7 +824,7 @@ static int hsw_pcm_close(struct snd_soc_component *component, struct sst_hsw *hsw = pdata->hsw; int ret, dai; - dai = mod_map[rtd->cpu_dai->id].dai_id; + dai = mod_map[asoc_rtd_to_cpu(rtd, 0)->id].dai_id; pcm_data = &pdata->pcm[dai][substream->stream]; mutex_lock(&pcm_data->mutex); @@ -923,9 +923,9 @@ static int hsw_pcm_new(struct snd_soc_component *component, hsw_pcm_hardware.buffer_bytes_max); } if (pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream) - priv_data->pcm[rtd->cpu_dai->id][SNDRV_PCM_STREAM_PLAYBACK].hsw_pcm = pcm; + priv_data->pcm[asoc_rtd_to_cpu(rtd, 0)->id][SNDRV_PCM_STREAM_PLAYBACK].hsw_pcm = pcm; if (pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream) - priv_data->pcm[rtd->cpu_dai->id][SNDRV_PCM_STREAM_CAPTURE].hsw_pcm = pcm; + priv_data->pcm[asoc_rtd_to_cpu(rtd, 0)->id][SNDRV_PCM_STREAM_CAPTURE].hsw_pcm = pcm; return 0; } diff --git a/sound/soc/intel/skylake/bxt-sst.c b/sound/soc/intel/skylake/bxt-sst.c index 92a82e6b5fe6..38b9d7494083 100644 --- a/sound/soc/intel/skylake/bxt-sst.c +++ b/sound/soc/intel/skylake/bxt-sst.c @@ -17,7 +17,6 @@ #include "skl.h" #define BXT_BASEFW_TIMEOUT 3000 -#define BXT_INIT_TIMEOUT 300 #define BXT_ROM_INIT_TIMEOUT 70 #define BXT_IPC_PURGE_FW 0x01004000 @@ -38,8 +37,6 @@ /* Delay before scheduling D0i3 entry */ #define BXT_D0I3_DELAY 5000 -#define BXT_FW_ROM_INIT_RETRY 3 - static unsigned int bxt_get_errorcode(struct sst_dsp *ctx) { return sst_dsp_shim_read(ctx, BXT_ADSP_ERROR_CODE); diff --git a/sound/soc/intel/skylake/cnl-sst.c b/sound/soc/intel/skylake/cnl-sst.c index 4f64f097e9ae..c6abcd5aa67b 100644 --- a/sound/soc/intel/skylake/cnl-sst.c +++ b/sound/soc/intel/skylake/cnl-sst.c @@ -57,18 +57,34 @@ static int cnl_prepare_fw(struct sst_dsp *ctx, const void *fwdata, u32 fwsize) ctx->dsp_ops.stream_tag = stream_tag; memcpy(ctx->dmab.area, fwdata, fwsize); + ret = skl_dsp_core_power_up(ctx, SKL_DSP_CORE0_MASK); + if (ret < 0) { + dev_err(ctx->dev, "dsp core0 power up failed\n"); + ret = -EIO; + goto base_fw_load_failed; + } + /* purge FW request */ sst_dsp_shim_write(ctx, CNL_ADSP_REG_HIPCIDR, CNL_ADSP_REG_HIPCIDR_BUSY | (CNL_IPC_PURGE | ((stream_tag - 1) << CNL_ROM_CTRL_DMA_ID))); - ret = cnl_dsp_enable_core(ctx, SKL_DSP_CORE0_MASK); + ret = skl_dsp_start_core(ctx, SKL_DSP_CORE0_MASK); if (ret < 0) { - dev_err(ctx->dev, "dsp boot core failed ret: %d\n", ret); + dev_err(ctx->dev, "Start dsp core failed ret: %d\n", ret); ret = -EIO; goto base_fw_load_failed; } + ret = sst_dsp_register_poll(ctx, CNL_ADSP_REG_HIPCIDA, + CNL_ADSP_REG_HIPCIDA_DONE, + CNL_ADSP_REG_HIPCIDA_DONE, + BXT_INIT_TIMEOUT, "HIPCIDA Done"); + if (ret < 0) { + dev_err(ctx->dev, "timeout for purge request: %d\n", ret); + goto base_fw_load_failed; + } + /* enable interrupt */ cnl_ipc_int_enable(ctx); cnl_ipc_op_int_enable(ctx); @@ -109,7 +125,7 @@ static int cnl_load_base_firmware(struct sst_dsp *ctx) { struct firmware stripped_fw; struct skl_dev *cnl = ctx->thread_context; - int ret; + int ret, i; if (!ctx->fw) { ret = request_firmware(&ctx->fw, ctx->fw_name, ctx->dev); @@ -131,12 +147,16 @@ static int cnl_load_base_firmware(struct sst_dsp *ctx) stripped_fw.size = ctx->fw->size; skl_dsp_strip_extended_manifest(&stripped_fw); - ret = cnl_prepare_fw(ctx, stripped_fw.data, stripped_fw.size); - if (ret < 0) { - dev_err(ctx->dev, "prepare firmware failed: %d\n", ret); - goto cnl_load_base_firmware_failed; + for (i = 0; i < BXT_FW_ROM_INIT_RETRY; i++) { + ret = cnl_prepare_fw(ctx, stripped_fw.data, stripped_fw.size); + if (!ret) + break; + dev_dbg(ctx->dev, "prepare firmware failed: %d\n", ret); } + if (ret < 0) + goto cnl_load_base_firmware_failed; + ret = sst_transfer_fw_host_dma(ctx); if (ret < 0) { dev_err(ctx->dev, "transfer firmware failed: %d\n", ret); @@ -158,6 +178,7 @@ static int cnl_load_base_firmware(struct sst_dsp *ctx) return 0; cnl_load_base_firmware_failed: + dev_err(ctx->dev, "firmware load failed: %d\n", ret); release_firmware(ctx->fw); ctx->fw = NULL; diff --git a/sound/soc/intel/skylake/skl-nhlt.c b/sound/soc/intel/skylake/skl-nhlt.c index 19f328d71f24..d9c8f5cb389e 100644 --- a/sound/soc/intel/skylake/skl-nhlt.c +++ b/sound/soc/intel/skylake/skl-nhlt.c @@ -182,7 +182,8 @@ void skl_nhlt_remove_sysfs(struct skl_dev *skl) { struct device *dev = &skl->pci->dev; - sysfs_remove_file(&dev->kobj, &dev_attr_platform_id.attr); + if (skl->nhlt) + sysfs_remove_file(&dev->kobj, &dev_attr_platform_id.attr); } /* diff --git a/sound/soc/intel/skylake/skl-pcm.c b/sound/soc/intel/skylake/skl-pcm.c index b99509675d29..89dcccdfb1cd 100644 --- a/sound/soc/intel/skylake/skl-pcm.c +++ b/sound/soc/intel/skylake/skl-pcm.c @@ -112,10 +112,7 @@ static void skl_set_suspend_active(struct snd_pcm_substream *substream, struct snd_soc_dapm_widget *w; struct skl_dev *skl = bus_to_skl(bus); - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) - w = dai->playback_widget; - else - w = dai->capture_widget; + w = snd_soc_dai_get_widget(dai, substream->stream); if (w->ignore_suspend && enable) skl->supend_active++; @@ -475,10 +472,7 @@ static int skl_pcm_trigger(struct snd_pcm_substream *substream, int cmd, if (!mconfig) return -EIO; - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) - w = dai->playback_widget; - else - w = dai->capture_widget; + w = snd_soc_dai_get_widget(dai, substream->stream); switch (cmd) { case SNDRV_PCM_TRIGGER_RESUME: @@ -551,7 +545,7 @@ static int skl_link_hw_params(struct snd_pcm_substream *substream, struct hdac_bus *bus = dev_get_drvdata(dai->dev); struct hdac_ext_stream *link_dev; struct snd_soc_pcm_runtime *rtd = snd_pcm_substream_chip(substream); - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct skl_pipe_params p_params = {0}; struct hdac_ext_link *link; int stream_tag; @@ -650,7 +644,7 @@ static int skl_link_hw_free(struct snd_pcm_substream *substream, link_dev->link_prepared = 0; - link = snd_hdac_ext_bus_get_link(bus, rtd->codec_dai->component->name); + link = snd_hdac_ext_bus_get_link(bus, asoc_rtd_to_codec(rtd, 0)->component->name); if (!link) return -EINVAL; @@ -1080,7 +1074,7 @@ static int skl_platform_soc_open(struct snd_soc_component *component, struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_dai_link *dai_link = rtd->dai_link; - dev_dbg(rtd->cpu_dai->dev, "In %s:%s\n", __func__, + dev_dbg(asoc_rtd_to_cpu(rtd, 0)->dev, "In %s:%s\n", __func__, dai_link->cpus->dai_name); snd_soc_set_runtime_hwparams(substream, &azx_pcm_hw); @@ -1232,7 +1226,7 @@ static u64 skl_adjust_codec_delay(struct snd_pcm_substream *substream, u64 nsec) { struct snd_soc_pcm_runtime *rtd = snd_pcm_substream_chip(substream); - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); u64 codec_frames, codec_nsecs; if (!codec_dai->driver->ops->delay) @@ -1287,7 +1281,7 @@ static int skl_platform_soc_get_time_info( static int skl_platform_soc_new(struct snd_soc_component *component, struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *dai = rtd->cpu_dai; + struct snd_soc_dai *dai = asoc_rtd_to_cpu(rtd, 0); struct hdac_bus *bus = dev_get_drvdata(dai->dev); struct snd_pcm *pcm = rtd->pcm; unsigned int size; diff --git a/sound/soc/intel/skylake/skl-sst-dsp.h b/sound/soc/intel/skylake/skl-sst-dsp.h index cdfec0fca577..1df9ef422f61 100644 --- a/sound/soc/intel/skylake/skl-sst-dsp.h +++ b/sound/soc/intel/skylake/skl-sst-dsp.h @@ -67,6 +67,8 @@ struct skl_dev; #define SKL_FW_INIT 0x1 #define SKL_FW_RFW_START 0xf +#define BXT_FW_ROM_INIT_RETRY 3 +#define BXT_INIT_TIMEOUT 300 #define SKL_ADSPIC_IPC 1 #define SKL_ADSPIS_IPC 1 diff --git a/sound/soc/intel/skylake/skl.c b/sound/soc/intel/skylake/skl.c index f755ca2484cf..63182bfd7941 100644 --- a/sound/soc/intel/skylake/skl.c +++ b/sound/soc/intel/skylake/skl.c @@ -130,6 +130,7 @@ static int skl_init_chip(struct hdac_bus *bus, bool full_reset) struct hdac_ext_link *hlink; int ret; + snd_hdac_set_codec_wakeup(bus, true); skl_enable_miscbdcge(bus->dev, false); ret = snd_hdac_bus_init_chip(bus, full_reset); @@ -138,6 +139,7 @@ static int skl_init_chip(struct hdac_bus *bus, bool full_reset) writel(0, hlink->ml_addr + AZX_REG_ML_LOSIDV); skl_enable_miscbdcge(bus->dev, true); + snd_hdac_set_codec_wakeup(bus, false); return ret; } @@ -359,7 +361,7 @@ static int skl_resume(struct device *dev) struct pci_dev *pci = to_pci_dev(dev); struct hdac_bus *bus = pci_get_drvdata(pci); struct skl_dev *skl = bus_to_skl(bus); - struct hdac_ext_link *hlink = NULL; + struct hdac_ext_link *hlink; int ret; /* @@ -481,13 +483,8 @@ static struct skl_ssp_clk skl_ssp_clks[] = { static struct snd_soc_acpi_mach *skl_find_hda_machine(struct skl_dev *skl, struct snd_soc_acpi_mach *machines) { - struct hdac_bus *bus = skl_to_bus(skl); struct snd_soc_acpi_mach *mach; - /* check if we have any codecs detected on bus */ - if (bus->codec_mask == 0) - return NULL; - /* point to common table */ mach = snd_soc_acpi_intel_hda_machines; @@ -636,6 +633,9 @@ static int skl_clock_device_register(struct skl_dev *skl) struct platform_device_info pdevinfo = {NULL}; struct skl_clk_pdata *clk_pdata; + if (!skl->nhlt) + return 0; + clk_pdata = devm_kzalloc(&skl->pci->dev, sizeof(*clk_pdata), GFP_KERNEL); if (!clk_pdata) @@ -794,7 +794,7 @@ static void skl_probe_work(struct work_struct *work) { struct skl_dev *skl = container_of(work, struct skl_dev, probe_work); struct hdac_bus *bus = skl_to_bus(skl); - struct hdac_ext_link *hlink = NULL; + struct hdac_ext_link *hlink; int err; if (IS_ENABLED(CONFIG_SND_SOC_HDAC_HDMI)) { @@ -803,6 +803,9 @@ static void skl_probe_work(struct work_struct *work) return; } + skl_init_pci(skl); + skl_dum_set(bus); + err = skl_init_chip(bus, true); if (err < 0) { dev_err(bus->dev, "Init chip failed with err: %d\n", err); @@ -918,8 +921,6 @@ static int skl_first_init(struct hdac_bus *bus) return -ENXIO; } - snd_hdac_bus_reset_link(bus, true); - snd_hdac_bus_parse_capabilities(bus); /* check if PPCAP exists */ @@ -967,11 +968,7 @@ static int skl_first_init(struct hdac_bus *bus) if (err < 0) return err; - /* initialize chip */ - skl_init_pci(skl); - skl_dum_set(bus); - - return skl_init_chip(bus, true); + return 0; } static int skl_probe(struct pci_dev *pci, @@ -1064,8 +1061,6 @@ static int skl_probe(struct pci_dev *pci, if (bus->mlcap) snd_hdac_ext_bus_get_ml_capabilities(bus); - snd_hdac_bus_stop_chip(bus); - /* create device for soc dmic */ err = skl_dmic_device_register(skl); if (err < 0) { @@ -1082,7 +1077,8 @@ static int skl_probe(struct pci_dev *pci, out_clk_free: skl_clock_device_unregister(skl); out_nhlt_free: - intel_nhlt_free(skl->nhlt); + if (skl->nhlt) + intel_nhlt_free(skl->nhlt); out_free: skl_free(bus); @@ -1131,7 +1127,8 @@ static void skl_remove(struct pci_dev *pci) skl_dmic_device_unregister(skl); skl_clock_device_unregister(skl); skl_nhlt_remove_sysfs(skl); - intel_nhlt_free(skl->nhlt); + if (skl->nhlt) + intel_nhlt_free(skl->nhlt); skl_free(bus); dev_set_drvdata(&pci->dev, NULL); } diff --git a/sound/soc/jz4740/jz4740-i2s.c b/sound/soc/jz4740/jz4740-i2s.c index 9d5405881209..6f6f8dad0356 100644 --- a/sound/soc/jz4740/jz4740-i2s.c +++ b/sound/soc/jz4740/jz4740-i2s.c @@ -49,12 +49,8 @@ #define JZ_AIC_CONF_FIFO_RX_THRESHOLD_OFFSET 12 #define JZ_AIC_CONF_FIFO_TX_THRESHOLD_OFFSET 8 -#define JZ4780_AIC_CONF_FIFO_RX_THRESHOLD_OFFSET 24 -#define JZ4780_AIC_CONF_FIFO_TX_THRESHOLD_OFFSET 16 -#define JZ4780_AIC_CONF_FIFO_RX_THRESHOLD_MASK \ - (0xf << JZ4780_AIC_CONF_FIFO_RX_THRESHOLD_OFFSET) -#define JZ4780_AIC_CONF_FIFO_TX_THRESHOLD_MASK \ - (0x1f << JZ4780_AIC_CONF_FIFO_TX_THRESHOLD_OFFSET) +#define JZ4760_AIC_CONF_FIFO_RX_THRESHOLD_OFFSET 24 +#define JZ4760_AIC_CONF_FIFO_TX_THRESHOLD_OFFSET 16 #define JZ_AIC_CTRL_OUTPUT_SAMPLE_SIZE_MASK (0x7 << 19) #define JZ_AIC_CTRL_INPUT_SAMPLE_SIZE_MASK (0x7 << 16) @@ -83,16 +79,23 @@ #define JZ_AIC_I2S_STATUS_BUSY BIT(2) #define JZ_AIC_CLK_DIV_MASK 0xf -#define I2SDIV_DV_SHIFT 8 +#define I2SDIV_DV_SHIFT 0 #define I2SDIV_DV_MASK (0xf << I2SDIV_DV_SHIFT) #define I2SDIV_IDV_SHIFT 8 #define I2SDIV_IDV_MASK (0xf << I2SDIV_IDV_SHIFT) enum jz47xx_i2s_version { JZ_I2S_JZ4740, + JZ_I2S_JZ4760, + JZ_I2S_JZ4770, JZ_I2S_JZ4780, }; +struct i2s_soc_info { + enum jz47xx_i2s_version version; + struct snd_soc_dai_driver *dai; +}; + struct jz4740_i2s { struct resource *mem; void __iomem *base; @@ -104,7 +107,7 @@ struct jz4740_i2s { struct snd_dmaengine_dai_dma_data playback_dma_data; struct snd_dmaengine_dai_dma_data capture_dma_data; - enum jz47xx_i2s_version version; + const struct i2s_soc_info *soc_info; }; static inline uint32_t jz4740_i2s_read(const struct jz4740_i2s *i2s, @@ -284,7 +287,7 @@ static int jz4740_i2s_hw_params(struct snd_pcm_substream *substream, ctrl &= ~JZ_AIC_CTRL_INPUT_SAMPLE_SIZE_MASK; ctrl |= sample_size << JZ_AIC_CTRL_INPUT_SAMPLE_SIZE_OFFSET; - if (i2s->version >= JZ_I2S_JZ4780) { + if (i2s->soc_info->version >= JZ_I2S_JZ4770) { div_reg &= ~I2SDIV_IDV_MASK; div_reg |= (div - 1) << I2SDIV_IDV_SHIFT; } else { @@ -398,9 +401,9 @@ static int jz4740_i2s_dai_probe(struct snd_soc_dai *dai) snd_soc_dai_init_dma_data(dai, &i2s->playback_dma_data, &i2s->capture_dma_data); - if (i2s->version >= JZ_I2S_JZ4780) { - conf = (7 << JZ4780_AIC_CONF_FIFO_RX_THRESHOLD_OFFSET) | - (8 << JZ4780_AIC_CONF_FIFO_TX_THRESHOLD_OFFSET) | + if (i2s->soc_info->version >= JZ_I2S_JZ4760) { + conf = (7 << JZ4760_AIC_CONF_FIFO_RX_THRESHOLD_OFFSET) | + (8 << JZ4760_AIC_CONF_FIFO_TX_THRESHOLD_OFFSET) | JZ_AIC_CONF_OVERFLOW_PLAY_LAST | JZ_AIC_CONF_I2S | JZ_AIC_CONF_INTERNAL_CODEC; @@ -457,7 +460,17 @@ static struct snd_soc_dai_driver jz4740_i2s_dai = { .ops = &jz4740_i2s_dai_ops, }; -static struct snd_soc_dai_driver jz4780_i2s_dai = { +static const struct i2s_soc_info jz4740_i2s_soc_info = { + .version = JZ_I2S_JZ4740, + .dai = &jz4740_i2s_dai, +}; + +static const struct i2s_soc_info jz4760_i2s_soc_info = { + .version = JZ_I2S_JZ4760, + .dai = &jz4740_i2s_dai, +}; + +static struct snd_soc_dai_driver jz4770_i2s_dai = { .probe = jz4740_i2s_dai_probe, .remove = jz4740_i2s_dai_remove, .playback = { @@ -475,6 +488,16 @@ static struct snd_soc_dai_driver jz4780_i2s_dai = { .ops = &jz4740_i2s_dai_ops, }; +static const struct i2s_soc_info jz4770_i2s_soc_info = { + .version = JZ_I2S_JZ4770, + .dai = &jz4770_i2s_dai, +}; + +static const struct i2s_soc_info jz4780_i2s_soc_info = { + .version = JZ_I2S_JZ4780, + .dai = &jz4770_i2s_dai, +}; + static const struct snd_soc_component_driver jz4740_i2s_component = { .name = "jz4740-i2s", .suspend = jz4740_i2s_suspend, @@ -483,8 +506,10 @@ static const struct snd_soc_component_driver jz4740_i2s_component = { #ifdef CONFIG_OF static const struct of_device_id jz4740_of_matches[] = { - { .compatible = "ingenic,jz4740-i2s", .data = (void *)JZ_I2S_JZ4740 }, - { .compatible = "ingenic,jz4780-i2s", .data = (void *)JZ_I2S_JZ4780 }, + { .compatible = "ingenic,jz4740-i2s", .data = &jz4740_i2s_soc_info }, + { .compatible = "ingenic,jz4760-i2s", .data = &jz4760_i2s_soc_info }, + { .compatible = "ingenic,jz4770-i2s", .data = &jz4770_i2s_soc_info }, + { .compatible = "ingenic,jz4780-i2s", .data = &jz4780_i2s_soc_info }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, jz4740_of_matches); @@ -492,45 +517,40 @@ MODULE_DEVICE_TABLE(of, jz4740_of_matches); static int jz4740_i2s_dev_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct jz4740_i2s *i2s; struct resource *mem; int ret; - i2s = devm_kzalloc(&pdev->dev, sizeof(*i2s), GFP_KERNEL); + i2s = devm_kzalloc(dev, sizeof(*i2s), GFP_KERNEL); if (!i2s) return -ENOMEM; - i2s->version = - (enum jz47xx_i2s_version)of_device_get_match_data(&pdev->dev); + i2s->soc_info = device_get_match_data(dev); mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2s->base = devm_ioremap_resource(&pdev->dev, mem); + i2s->base = devm_ioremap_resource(dev, mem); if (IS_ERR(i2s->base)) return PTR_ERR(i2s->base); i2s->phys_base = mem->start; - i2s->clk_aic = devm_clk_get(&pdev->dev, "aic"); + i2s->clk_aic = devm_clk_get(dev, "aic"); if (IS_ERR(i2s->clk_aic)) return PTR_ERR(i2s->clk_aic); - i2s->clk_i2s = devm_clk_get(&pdev->dev, "i2s"); + i2s->clk_i2s = devm_clk_get(dev, "i2s"); if (IS_ERR(i2s->clk_i2s)) return PTR_ERR(i2s->clk_i2s); platform_set_drvdata(pdev, i2s); - if (i2s->version == JZ_I2S_JZ4780) - ret = devm_snd_soc_register_component(&pdev->dev, - &jz4740_i2s_component, &jz4780_i2s_dai, 1); - else - ret = devm_snd_soc_register_component(&pdev->dev, - &jz4740_i2s_component, &jz4740_i2s_dai, 1); - + ret = devm_snd_soc_register_component(dev, &jz4740_i2s_component, + i2s->soc_info->dai, 1); if (ret) return ret; - return devm_snd_dmaengine_pcm_register(&pdev->dev, NULL, + return devm_snd_dmaengine_pcm_register(dev, NULL, SND_DMAENGINE_PCM_FLAG_COMPAT); } diff --git a/sound/soc/kirkwood/armada-370-db.c b/sound/soc/kirkwood/armada-370-db.c index 8c3c808bda9a..4f66b011f1b4 100644 --- a/sound/soc/kirkwood/armada-370-db.c +++ b/sound/soc/kirkwood/armada-370-db.c @@ -19,7 +19,7 @@ static int a370db_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); unsigned int freq; switch (params_rate(params)) { diff --git a/sound/soc/kirkwood/kirkwood-dma.c b/sound/soc/kirkwood/kirkwood-dma.c index f882b4003edf..e037826b2451 100644 --- a/sound/soc/kirkwood/kirkwood-dma.c +++ b/sound/soc/kirkwood/kirkwood-dma.c @@ -20,7 +20,7 @@ static struct kirkwood_dma_data *kirkwood_priv(struct snd_pcm_substream *subs) { struct snd_soc_pcm_runtime *soc_runtime = subs->private_data; - return snd_soc_dai_get_drvdata(soc_runtime->cpu_dai); + return snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(soc_runtime, 0)); } static const struct snd_pcm_hardware kirkwood_dma_snd_hw = { diff --git a/sound/soc/mediatek/common/mtk-afe-fe-dai.c b/sound/soc/mediatek/common/mtk-afe-fe-dai.c index 4254f3a954dd..375e3b492922 100644 --- a/sound/soc/mediatek/common/mtk-afe-fe-dai.c +++ b/sound/soc/mediatek/common/mtk-afe-fe-dai.c @@ -40,7 +40,7 @@ int mtk_afe_fe_startup(struct snd_pcm_substream *substream, struct snd_soc_pcm_runtime *rtd = substream->private_data; struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai); struct snd_pcm_runtime *runtime = substream->runtime; - int memif_num = rtd->cpu_dai->id; + int memif_num = asoc_rtd_to_cpu(rtd, 0)->id; struct mtk_base_afe_memif *memif = &afe->memif[memif_num]; const struct snd_pcm_hardware *mtk_afe_hardware = afe->mtk_afe_hardware; int ret; @@ -100,7 +100,7 @@ void mtk_afe_fe_shutdown(struct snd_pcm_substream *substream, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai); - struct mtk_base_afe_memif *memif = &afe->memif[rtd->cpu_dai->id]; + struct mtk_base_afe_memif *memif = &afe->memif[asoc_rtd_to_cpu(rtd, 0)->id]; int irq_id; irq_id = memif->irq_usage; @@ -122,7 +122,7 @@ int mtk_afe_fe_hw_params(struct snd_pcm_substream *substream, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai); - int id = rtd->cpu_dai->id; + int id = asoc_rtd_to_cpu(rtd, 0)->id; struct mtk_base_afe_memif *memif = &afe->memif[id]; int ret; unsigned int channels = params_channels(params); @@ -199,7 +199,7 @@ int mtk_afe_fe_trigger(struct snd_pcm_substream *substream, int cmd, struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_pcm_runtime * const runtime = substream->runtime; struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai); - int id = rtd->cpu_dai->id; + int id = asoc_rtd_to_cpu(rtd, 0)->id; struct mtk_base_afe_memif *memif = &afe->memif[id]; struct mtk_base_afe_irq *irqs = &afe->irqs[memif->irq_usage]; const struct mtk_base_irq_data *irq_data = irqs->irq_data; @@ -265,7 +265,7 @@ int mtk_afe_fe_prepare(struct snd_pcm_substream *substream, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai); - int id = rtd->cpu_dai->id; + int id = asoc_rtd_to_cpu(rtd, 0)->id; int pbuf_size; if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { diff --git a/sound/soc/mediatek/common/mtk-afe-platform-driver.c b/sound/soc/mediatek/common/mtk-afe-platform-driver.c index 44dfef713905..0a1a65c86f0e 100644 --- a/sound/soc/mediatek/common/mtk-afe-platform-driver.c +++ b/sound/soc/mediatek/common/mtk-afe-platform-driver.c @@ -82,7 +82,7 @@ snd_pcm_uframes_t mtk_afe_pcm_pointer(struct snd_soc_component *component, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct mtk_base_afe *afe = snd_soc_component_get_drvdata(component); - struct mtk_base_afe_memif *memif = &afe->memif[rtd->cpu_dai->id]; + struct mtk_base_afe_memif *memif = &afe->memif[asoc_rtd_to_cpu(rtd, 0)->id]; const struct mtk_base_memif_data *memif_data = memif->data; struct regmap *regmap = afe->regmap; struct device *dev = afe->dev; diff --git a/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c b/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c index 488603a0c4b1..f0250b0dd734 100644 --- a/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c +++ b/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c @@ -497,7 +497,7 @@ static int mt2701_memif_fs(struct snd_pcm_substream *substream, struct snd_soc_pcm_runtime *rtd = substream->private_data; int fs; - if (rtd->cpu_dai->id != MT2701_MEMIF_ULBT) + if (asoc_rtd_to_cpu(rtd, 0)->id != MT2701_MEMIF_ULBT) fs = mt2701_afe_i2s_fs(rate); else fs = (rate == 16000 ? 1 : 0); diff --git a/sound/soc/mediatek/mt2701/mt2701-cs42448.c b/sound/soc/mediatek/mt2701/mt2701-cs42448.c index b6941796efca..c47af9b6949b 100644 --- a/sound/soc/mediatek/mt2701/mt2701-cs42448.c +++ b/sound/soc/mediatek/mt2701/mt2701-cs42448.c @@ -128,8 +128,8 @@ static int mt2701_cs42448_be_ops_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); unsigned int mclk_rate; unsigned int rate = params_rate(params); unsigned int div_mclk_over_bck = rate > 192000 ? 2 : 4; diff --git a/sound/soc/mediatek/mt2701/mt2701-wm8960.c b/sound/soc/mediatek/mt2701/mt2701-wm8960.c index 8c4c89e4c616..0122e7df067f 100644 --- a/sound/soc/mediatek/mt2701/mt2701-wm8960.c +++ b/sound/soc/mediatek/mt2701/mt2701-wm8960.c @@ -25,8 +25,8 @@ static int mt2701_wm8960_be_ops_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int mclk_rate; unsigned int rate = params_rate(params); unsigned int div_mclk_over_bck = rate > 192000 ? 2 : 4; diff --git a/sound/soc/mediatek/mt6797/mt6797-afe-pcm.c b/sound/soc/mediatek/mt6797/mt6797-afe-pcm.c index 378bfc16ef52..7f930556d961 100644 --- a/sound/soc/mediatek/mt6797/mt6797-afe-pcm.c +++ b/sound/soc/mediatek/mt6797/mt6797-afe-pcm.c @@ -143,7 +143,7 @@ static int mt6797_memif_fs(struct snd_pcm_substream *substream, struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); struct mtk_base_afe *afe = snd_soc_component_get_drvdata(component); - int id = rtd->cpu_dai->id; + int id = asoc_rtd_to_cpu(rtd, 0)->id; return mt6797_rate_transform(afe->dev, rate, id); } diff --git a/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c b/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c index 461e4de8c918..1e3f2d786066 100644 --- a/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c +++ b/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c @@ -485,7 +485,7 @@ static int mt8173_memif_fs(struct snd_pcm_substream *substream, struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); struct mtk_base_afe *afe = snd_soc_component_get_drvdata(component); - struct mtk_base_afe_memif *memif = &afe->memif[rtd->cpu_dai->id]; + struct mtk_base_afe_memif *memif = &afe->memif[asoc_rtd_to_cpu(rtd, 0)->id]; int fs; if (memif->data->id == MT8173_AFE_MEMIF_DAI || diff --git a/sound/soc/mediatek/mt8173/mt8173-max98090.c b/sound/soc/mediatek/mt8173/mt8173-max98090.c index 22c00600c999..37693d354e66 100644 --- a/sound/soc/mediatek/mt8173/mt8173-max98090.c +++ b/sound/soc/mediatek/mt8173/mt8173-max98090.c @@ -53,7 +53,7 @@ static int mt8173_max98090_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); return snd_soc_dai_set_sysclk(codec_dai, 0, params_rate(params) * 256, SND_SOC_CLOCK_IN); @@ -67,7 +67,7 @@ static int mt8173_max98090_init(struct snd_soc_pcm_runtime *runtime) { int ret; struct snd_soc_card *card = runtime->card; - struct snd_soc_component *component = runtime->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(runtime, 0)->component; /* enable jack detection */ ret = snd_soc_card_jack_new(card, "Headphone", SND_JACK_HEADPHONE, diff --git a/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c b/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c index 2e1e61d8f127..51009a172777 100644 --- a/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c +++ b/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c @@ -47,7 +47,7 @@ static int mt8173_rt5650_rt5514_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *codec_dai; int i, ret; - for_each_rtd_codec_dai(rtd, i, codec_dai) { + for_each_rtd_codec_dais(rtd, i, codec_dai) { /* pll from mclk 12.288M */ ret = snd_soc_dai_set_pll(codec_dai, 0, 0, MCLK_FOR_CODECS, params_rate(params) * 512); @@ -73,7 +73,7 @@ static struct snd_soc_jack mt8173_rt5650_rt5514_jack; static int mt8173_rt5650_rt5514_init(struct snd_soc_pcm_runtime *runtime) { struct snd_soc_card *card = runtime->card; - struct snd_soc_component *component = runtime->codec_dais[0]->component; + struct snd_soc_component *component = asoc_rtd_to_codec(runtime, 0)->component; int ret; rt5645_sel_asrc_clk_src(component, diff --git a/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c b/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c index ebcc0b86286b..247ac7690805 100644 --- a/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c +++ b/sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c @@ -51,7 +51,7 @@ static int mt8173_rt5650_rt5676_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *codec_dai; int i, ret; - for_each_rtd_codec_dai(rtd, i, codec_dai) { + for_each_rtd_codec_dais(rtd, i, codec_dai) { /* pll from mclk 12.288M */ ret = snd_soc_dai_set_pll(codec_dai, 0, 0, MCLK_FOR_CODECS, params_rate(params) * 512); @@ -77,8 +77,8 @@ static struct snd_soc_jack mt8173_rt5650_rt5676_jack; static int mt8173_rt5650_rt5676_init(struct snd_soc_pcm_runtime *runtime) { struct snd_soc_card *card = runtime->card; - struct snd_soc_component *component = runtime->codec_dais[0]->component; - struct snd_soc_component *component_sub = runtime->codec_dais[1]->component; + struct snd_soc_component *component = asoc_rtd_to_codec(runtime, 0)->component; + struct snd_soc_component *component_sub = asoc_rtd_to_codec(runtime, 1)->component; int ret; rt5645_sel_asrc_clk_src(component, diff --git a/sound/soc/mediatek/mt8173/mt8173-rt5650.c b/sound/soc/mediatek/mt8173/mt8173-rt5650.c index ef6f23675286..2065c94dbf99 100644 --- a/sound/soc/mediatek/mt8173/mt8173-rt5650.c +++ b/sound/soc/mediatek/mt8173/mt8173-rt5650.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "../../codecs/rt5645.h" #define MCLK_FOR_CODECS 12288000 @@ -77,7 +78,7 @@ static int mt8173_rt5650_hw_params(struct snd_pcm_substream *substream, break; } - for_each_rtd_codec_dai(rtd, i, codec_dai) { + for_each_rtd_codec_dais(rtd, i, codec_dai) { /* pll from mclk */ ret = snd_soc_dai_set_pll(codec_dai, 0, 0, mclk_clock, params_rate(params) * 512); @@ -98,13 +99,13 @@ static const struct snd_soc_ops mt8173_rt5650_ops = { .hw_params = mt8173_rt5650_hw_params, }; -static struct snd_soc_jack mt8173_rt5650_jack; +static struct snd_soc_jack mt8173_rt5650_jack, mt8173_rt5650_hdmi_jack; static int mt8173_rt5650_init(struct snd_soc_pcm_runtime *runtime) { struct snd_soc_card *card = runtime->card; - struct snd_soc_component *component = runtime->codec_dais[0]->component; - const char *codec_capture_dai = runtime->codec_dais[1]->name; + struct snd_soc_component *component = asoc_rtd_to_codec(runtime, 0)->component; + const char *codec_capture_dai = asoc_rtd_to_codec(runtime, 1)->name; int ret; rt5645_sel_asrc_clk_src(component, @@ -144,6 +145,19 @@ static int mt8173_rt5650_init(struct snd_soc_pcm_runtime *runtime) &mt8173_rt5650_jack); } +static int mt8173_rt5650_hdmi_init(struct snd_soc_pcm_runtime *rtd) +{ + int ret; + + ret = snd_soc_card_jack_new(rtd->card, "HDMI Jack", SND_JACK_LINEOUT, + &mt8173_rt5650_hdmi_jack, NULL, 0); + if (ret) + return ret; + + return hdmi_codec_set_jack_detect(asoc_rtd_to_codec(rtd, 0)->component, + &mt8173_rt5650_hdmi_jack); +} + enum { DAI_LINK_PLAYBACK, DAI_LINK_CAPTURE, @@ -222,6 +236,7 @@ static struct snd_soc_dai_link mt8173_rt5650_dais[] = { .name = "HDMI BE", .no_pcm = 1, .dpcm_playback = 1, + .init = mt8173_rt5650_hdmi_init, SND_SOC_DAILINK_REG(hdmi_be), }, }; diff --git a/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c b/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c index 6e2270bbb10e..c8ded53bde1d 100644 --- a/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c +++ b/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c @@ -146,7 +146,7 @@ static int mt8183_memif_fs(struct snd_pcm_substream *substream, struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); struct mtk_base_afe *afe = snd_soc_component_get_drvdata(component); - int id = rtd->cpu_dai->id; + int id = asoc_rtd_to_cpu(rtd, 0)->id; return mt8183_rate_transform(afe->dev, rate, id); } diff --git a/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c index c65493721e90..5b3dfa79b4ae 100644 --- a/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c +++ b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c @@ -16,7 +16,9 @@ #include "../../codecs/da7219-aad.h" #include "../../codecs/da7219.h" -static struct snd_soc_jack headset_jack; +struct mt8183_da7219_max98357_priv { + struct snd_soc_jack headset_jack; +}; static int mt8183_mt6358_i2s_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) @@ -26,7 +28,7 @@ static int mt8183_mt6358_i2s_hw_params(struct snd_pcm_substream *substream, unsigned int mclk_fs_ratio = 128; unsigned int mclk_fs = rate * mclk_fs_ratio; - return snd_soc_dai_set_sysclk(rtd->cpu_dai, + return snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(rtd, 0), 0, mclk_fs, SND_SOC_CLOCK_OUT); } @@ -38,19 +40,19 @@ static int mt8183_da7219_i2s_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *codec_dai; unsigned int rate = params_rate(params); unsigned int mclk_fs_ratio = 256; unsigned int mclk_fs = rate * mclk_fs_ratio; unsigned int freq; int ret = 0, j; - ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, 0, + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(rtd, 0), 0, mclk_fs, SND_SOC_CLOCK_OUT); if (ret < 0) dev_err(rtd->dev, "failed to set cpu dai sysclk\n"); - for (j = 0; j < rtd->num_codecs; j++) { - struct snd_soc_dai *codec_dai = rtd->codec_dais[j]; + for_each_rtd_codec_dais(rtd, j, codec_dai) { if (!strcmp(codec_dai->component->name, "da7219.5-001a")) { ret = snd_soc_dai_set_sysclk(codec_dai, @@ -80,10 +82,10 @@ static int mt8183_da7219_i2s_hw_params(struct snd_pcm_substream *substream, static int mt8183_da7219_hw_free(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *codec_dai; int ret = 0, j; - for (j = 0; j < rtd->num_codecs; j++) { - struct snd_soc_dai *codec_dai = rtd->codec_dais[j]; + for_each_rtd_codec_dais(rtd, j, codec_dai) { if (!strcmp(codec_dai->component->name, "da7219.5-001a")) { ret = snd_soc_dai_set_pll(codec_dai, @@ -116,6 +118,46 @@ static int mt8183_i2s_hw_params_fixup(struct snd_soc_pcm_runtime *rtd, return 0; } +static int +mt8183_da7219_max98357_bt_sco_startup( + struct snd_pcm_substream *substream) +{ + static const unsigned int rates[] = { + 8000, 16000 + }; + static const struct snd_pcm_hw_constraint_list constraints_rates = { + .count = ARRAY_SIZE(rates), + .list = rates, + .mask = 0, + }; + static const unsigned int channels[] = { + 1, + }; + static const struct snd_pcm_hw_constraint_list constraints_channels = { + .count = ARRAY_SIZE(channels), + .list = channels, + .mask = 0, + }; + + struct snd_pcm_runtime *runtime = substream->runtime; + + snd_pcm_hw_constraint_list(runtime, 0, + SNDRV_PCM_HW_PARAM_RATE, &constraints_rates); + runtime->hw.channels_max = 1; + snd_pcm_hw_constraint_list(runtime, 0, + SNDRV_PCM_HW_PARAM_CHANNELS, + &constraints_channels); + + runtime->hw.formats = SNDRV_PCM_FMTBIT_S16_LE; + snd_pcm_hw_constraint_msbits(runtime, 0, 16, 16); + + return 0; +} + +static const struct snd_soc_ops mt8183_da7219_max98357_bt_sco_ops = { + .startup = mt8183_da7219_max98357_bt_sco_startup, +}; + /* FE */ SND_SOC_DAILINK_DEFS(playback1, DAILINK_COMP_ARRAY(COMP_CPU("DL1")), @@ -222,6 +264,7 @@ static struct snd_soc_dai_link mt8183_da7219_max98357_dai_links[] = { SND_SOC_DPCM_TRIGGER_PRE}, .dynamic = 1, .dpcm_playback = 1, + .ops = &mt8183_da7219_max98357_bt_sco_ops, SND_SOC_DAILINK_REG(playback2), }, { @@ -240,6 +283,7 @@ static struct snd_soc_dai_link mt8183_da7219_max98357_dai_links[] = { SND_SOC_DPCM_TRIGGER_PRE}, .dynamic = 1, .dpcm_capture = 1, + .ops = &mt8183_da7219_max98357_bt_sco_ops, SND_SOC_DAILINK_REG(capture1), }, { @@ -351,8 +395,12 @@ static struct snd_soc_dai_link mt8183_da7219_max98357_dai_links[] = { { .name = "TDM", .no_pcm = 1, + .dai_fmt = SND_SOC_DAIFMT_I2S | + SND_SOC_DAIFMT_IB_IF | + SND_SOC_DAIFMT_CBM_CFM, .dpcm_playback = 1, .ignore_suspend = 1, + .be_hw_params_fixup = mt8183_i2s_hw_params_fixup, SND_SOC_DAILINK_REG(tdm), }, }; @@ -372,9 +420,31 @@ static struct snd_soc_codec_conf mt6358_codec_conf[] = { }, }; +static const struct snd_kcontrol_new mt8183_da7219_max98357_snd_controls[] = { + SOC_DAPM_PIN_SWITCH("Speakers"), +}; + +static const +struct snd_soc_dapm_widget mt8183_da7219_max98357_dapm_widgets[] = { + SND_SOC_DAPM_SPK("Speakers", NULL), + SND_SOC_DAPM_PINCTRL("TDM_OUT_PINCTRL", + "aud_tdm_out_on", "aud_tdm_out_off"), +}; + +static const struct snd_soc_dapm_route mt8183_da7219_max98357_dapm_routes[] = { + {"Speakers", NULL, "Speaker"}, + {"I2S Playback", NULL, "TDM_OUT_PINCTRL"}, +}; + static struct snd_soc_card mt8183_da7219_max98357_card = { .name = "mt8183_da7219_max98357", .owner = THIS_MODULE, + .controls = mt8183_da7219_max98357_snd_controls, + .num_controls = ARRAY_SIZE(mt8183_da7219_max98357_snd_controls), + .dapm_widgets = mt8183_da7219_max98357_dapm_widgets, + .num_dapm_widgets = ARRAY_SIZE(mt8183_da7219_max98357_dapm_widgets), + .dapm_routes = mt8183_da7219_max98357_dapm_routes, + .num_dapm_routes = ARRAY_SIZE(mt8183_da7219_max98357_dapm_routes), .dai_link = mt8183_da7219_max98357_dai_links, .num_links = ARRAY_SIZE(mt8183_da7219_max98357_dai_links), .aux_dev = &mt8183_da7219_max98357_headset_dev, @@ -387,6 +457,8 @@ static int mt8183_da7219_max98357_headset_init(struct snd_soc_component *component) { int ret; + struct mt8183_da7219_max98357_priv *priv = + snd_soc_card_get_drvdata(component->card); /* Enable Headset and 4 Buttons Jack detection */ ret = snd_soc_card_jack_new(&mt8183_da7219_max98357_card, @@ -394,12 +466,12 @@ mt8183_da7219_max98357_headset_init(struct snd_soc_component *component) SND_JACK_HEADSET | SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3, - &headset_jack, + &priv->headset_jack, NULL, 0); if (ret) return ret; - da7219_aad_jack_det(component, &headset_jack); + da7219_aad_jack_det(component, &priv->headset_jack); return ret; } @@ -409,7 +481,8 @@ static int mt8183_da7219_max98357_dev_probe(struct platform_device *pdev) struct snd_soc_card *card = &mt8183_da7219_max98357_card; struct device_node *platform_node; struct snd_soc_dai_link *dai_link; - struct pinctrl *default_pins; + struct mt8183_da7219_max98357_priv *priv; + struct pinctrl *pinctrl; int ret, i; card->dev = &pdev->dev; @@ -436,22 +509,21 @@ static int mt8183_da7219_max98357_dev_probe(struct platform_device *pdev) return -EINVAL; } - ret = devm_snd_soc_register_card(&pdev->dev, card); - if (ret) { - dev_err(&pdev->dev, "%s snd_soc_register_card fail %d\n", + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + snd_soc_card_set_drvdata(card, priv); + + pinctrl = devm_pinctrl_get_select(&pdev->dev, PINCTRL_STATE_DEFAULT); + if (IS_ERR(pinctrl)) { + ret = PTR_ERR(pinctrl); + dev_err(&pdev->dev, "%s failed to select default state %d\n", __func__, ret); return ret; } - default_pins = - devm_pinctrl_get_select(&pdev->dev, PINCTRL_STATE_DEFAULT); - if (IS_ERR(default_pins)) { - dev_err(&pdev->dev, "%s set pins failed\n", - __func__); - return PTR_ERR(default_pins); - } - - return ret; + return devm_snd_soc_register_card(&pdev->dev, card); } #ifdef CONFIG_OF @@ -478,4 +550,3 @@ MODULE_DESCRIPTION("MT8183-DA7219-MAX98357 ALSA SoC machine driver"); MODULE_AUTHOR("Shunli Wang "); MODULE_LICENSE("GPL v2"); MODULE_ALIAS("mt8183_da7219_max98357 soc card"); - diff --git a/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c index 0555f7d73d05..1fca8df109b4 100644 --- a/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c +++ b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c @@ -41,7 +41,7 @@ static int mt8183_mt6358_i2s_hw_params(struct snd_pcm_substream *substream, unsigned int mclk_fs_ratio = 128; unsigned int mclk_fs = rate * mclk_fs_ratio; - return snd_soc_dai_set_sysclk(rtd->cpu_dai, + return snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(rtd, 0), 0, mclk_fs, SND_SOC_CLOCK_OUT); } diff --git a/sound/soc/meson/Kconfig b/sound/soc/meson/Kconfig index 2e3676147cea..8b6295283989 100644 --- a/sound/soc/meson/Kconfig +++ b/sound/soc/meson/Kconfig @@ -2,6 +2,16 @@ menu "ASoC support for Amlogic platforms" depends on ARCH_MESON || COMPILE_TEST +config SND_MESON_AIU + tristate "Amlogic AIU" + select SND_MESON_CODEC_GLUE + select SND_PCM_IEC958 + imply SND_SOC_MESON_T9015 + imply SND_SOC_HDMI_CODEC if DRM_MESON_DW_HDMI + help + Select Y or M to add support for the Audio output subsystem found + in the Amlogic Meson8, Meson8b and GX SoC families + config SND_MESON_AXG_FIFO tristate select REGMAP_MMIO @@ -50,6 +60,7 @@ config SND_MESON_AXG_TDMOUT config SND_MESON_AXG_SOUND_CARD tristate "Amlogic AXG Sound Card Support" select SND_MESON_AXG_TDM_INTERFACE + select SND_MESON_CARD_UTILS imply SND_MESON_AXG_FRDDR imply SND_MESON_AXG_TODDR imply SND_MESON_AXG_TDMIN @@ -85,11 +96,41 @@ config SND_MESON_AXG_PDM Select Y or M to add support for PDM input embedded in the Amlogic AXG SoC family +config SND_MESON_CARD_UTILS + tristate + +config SND_MESON_CODEC_GLUE + tristate + +config SND_MESON_GX_SOUND_CARD + tristate "Amlogic GX Sound Card Support" + select SND_MESON_CARD_UTILS + imply SND_MESON_AIU + help + Select Y or M to add support for the GXBB/GXL SoC sound card + +config SND_MESON_G12A_TOACODEC + tristate "Amlogic G12A To Internal DAC Control Support" + select SND_MESON_CODEC_GLUE + select REGMAP_MMIO + imply SND_SOC_MESON_T9015 + help + Select Y or M to add support for the internal audio DAC on the + g12a SoC family + config SND_MESON_G12A_TOHDMITX tristate "Amlogic G12A To HDMI TX Control Support" select REGMAP_MMIO + select SND_MESON_CODEC_GLUE imply SND_SOC_HDMI_CODEC help Select Y or M to add support for HDMI audio on the g12a SoC family + +config SND_SOC_MESON_T9015 + tristate "Amlogic T9015 DAC" + select REGMAP_MMIO + help + Say Y or M if you want to add support for the internal DAC found + on GXL, G12 and SM1 SoC family. endmenu diff --git a/sound/soc/meson/Makefile b/sound/soc/meson/Makefile index 1a8b1470ed84..e446bc980481 100644 --- a/sound/soc/meson/Makefile +++ b/sound/soc/meson/Makefile @@ -1,5 +1,13 @@ # SPDX-License-Identifier: (GPL-2.0 OR MIT) +snd-soc-meson-aiu-objs := aiu.o +snd-soc-meson-aiu-objs += aiu-acodec-ctrl.o +snd-soc-meson-aiu-objs += aiu-codec-ctrl.o +snd-soc-meson-aiu-objs += aiu-encoder-i2s.o +snd-soc-meson-aiu-objs += aiu-encoder-spdif.o +snd-soc-meson-aiu-objs += aiu-fifo.o +snd-soc-meson-aiu-objs += aiu-fifo-i2s.o +snd-soc-meson-aiu-objs += aiu-fifo-spdif.o snd-soc-meson-axg-fifo-objs := axg-fifo.o snd-soc-meson-axg-frddr-objs := axg-frddr.o snd-soc-meson-axg-toddr-objs := axg-toddr.o @@ -11,8 +19,14 @@ snd-soc-meson-axg-sound-card-objs := axg-card.o snd-soc-meson-axg-spdifin-objs := axg-spdifin.o snd-soc-meson-axg-spdifout-objs := axg-spdifout.o snd-soc-meson-axg-pdm-objs := axg-pdm.o +snd-soc-meson-card-utils-objs := meson-card-utils.o +snd-soc-meson-codec-glue-objs := meson-codec-glue.o +snd-soc-meson-gx-sound-card-objs := gx-card.o +snd-soc-meson-g12a-toacodec-objs := g12a-toacodec.o snd-soc-meson-g12a-tohdmitx-objs := g12a-tohdmitx.o +snd-soc-meson-t9015-objs := t9015.o +obj-$(CONFIG_SND_MESON_AIU) += snd-soc-meson-aiu.o obj-$(CONFIG_SND_MESON_AXG_FIFO) += snd-soc-meson-axg-fifo.o obj-$(CONFIG_SND_MESON_AXG_FRDDR) += snd-soc-meson-axg-frddr.o obj-$(CONFIG_SND_MESON_AXG_TODDR) += snd-soc-meson-axg-toddr.o @@ -24,4 +38,9 @@ obj-$(CONFIG_SND_MESON_AXG_SOUND_CARD) += snd-soc-meson-axg-sound-card.o obj-$(CONFIG_SND_MESON_AXG_SPDIFIN) += snd-soc-meson-axg-spdifin.o obj-$(CONFIG_SND_MESON_AXG_SPDIFOUT) += snd-soc-meson-axg-spdifout.o obj-$(CONFIG_SND_MESON_AXG_PDM) += snd-soc-meson-axg-pdm.o +obj-$(CONFIG_SND_MESON_CARD_UTILS) += snd-soc-meson-card-utils.o +obj-$(CONFIG_SND_MESON_CODEC_GLUE) += snd-soc-meson-codec-glue.o +obj-$(CONFIG_SND_MESON_GX_SOUND_CARD) += snd-soc-meson-gx-sound-card.o +obj-$(CONFIG_SND_MESON_G12A_TOACODEC) += snd-soc-meson-g12a-toacodec.o obj-$(CONFIG_SND_MESON_G12A_TOHDMITX) += snd-soc-meson-g12a-tohdmitx.o +obj-$(CONFIG_SND_SOC_MESON_T9015) += snd-soc-meson-t9015.o diff --git a/sound/soc/meson/aiu-acodec-ctrl.c b/sound/soc/meson/aiu-acodec-ctrl.c new file mode 100644 index 000000000000..7078197e0cc5 --- /dev/null +++ b/sound/soc/meson/aiu-acodec-ctrl.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include + +#include +#include "aiu.h" +#include "meson-codec-glue.h" + +#define CTRL_DIN_EN 15 +#define CTRL_CLK_INV BIT(14) +#define CTRL_LRCLK_INV BIT(13) +#define CTRL_I2S_IN_BCLK_SRC BIT(11) +#define CTRL_DIN_LRCLK_SRC_SHIFT 6 +#define CTRL_DIN_LRCLK_SRC (0x3 << CTRL_DIN_LRCLK_SRC_SHIFT) +#define CTRL_BCLK_MCLK_SRC GENMASK(5, 4) +#define CTRL_DIN_SKEW GENMASK(3, 2) +#define CTRL_I2S_OUT_LANE_SRC 0 + +#define AIU_ACODEC_OUT_CHMAX 2 + +static const char * const aiu_acodec_ctrl_mux_texts[] = { + "DISABLED", "I2S", "PCM", +}; + +static int aiu_acodec_ctrl_mux_put_enum(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = + snd_soc_dapm_kcontrol_component(kcontrol); + struct snd_soc_dapm_context *dapm = + snd_soc_dapm_kcontrol_dapm(kcontrol); + struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; + unsigned int mux, changed; + + mux = snd_soc_enum_item_to_val(e, ucontrol->value.enumerated.item[0]); + changed = snd_soc_component_test_bits(component, e->reg, + CTRL_DIN_LRCLK_SRC, + FIELD_PREP(CTRL_DIN_LRCLK_SRC, + mux)); + + if (!changed) + return 0; + + /* Force disconnect of the mux while updating */ + snd_soc_dapm_mux_update_power(dapm, kcontrol, 0, NULL, NULL); + + snd_soc_component_update_bits(component, e->reg, + CTRL_DIN_LRCLK_SRC | + CTRL_BCLK_MCLK_SRC, + FIELD_PREP(CTRL_DIN_LRCLK_SRC, mux) | + FIELD_PREP(CTRL_BCLK_MCLK_SRC, mux)); + + snd_soc_dapm_mux_update_power(dapm, kcontrol, mux, e, NULL); + + return 0; +} + +static SOC_ENUM_SINGLE_DECL(aiu_acodec_ctrl_mux_enum, AIU_ACODEC_CTRL, + CTRL_DIN_LRCLK_SRC_SHIFT, + aiu_acodec_ctrl_mux_texts); + +static const struct snd_kcontrol_new aiu_acodec_ctrl_mux = + SOC_DAPM_ENUM_EXT("ACodec Source", aiu_acodec_ctrl_mux_enum, + snd_soc_dapm_get_enum_double, + aiu_acodec_ctrl_mux_put_enum); + +static const struct snd_kcontrol_new aiu_acodec_ctrl_out_enable = + SOC_DAPM_SINGLE_AUTODISABLE("Switch", AIU_ACODEC_CTRL, + CTRL_DIN_EN, 1, 0); + +static const struct snd_soc_dapm_widget aiu_acodec_ctrl_widgets[] = { + SND_SOC_DAPM_MUX("ACODEC SRC", SND_SOC_NOPM, 0, 0, + &aiu_acodec_ctrl_mux), + SND_SOC_DAPM_SWITCH("ACODEC OUT EN", SND_SOC_NOPM, 0, 0, + &aiu_acodec_ctrl_out_enable), +}; + +static int aiu_acodec_ctrl_input_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct meson_codec_glue_input *data; + int ret; + + ret = meson_codec_glue_input_hw_params(substream, params, dai); + if (ret) + return ret; + + /* The glue will provide 1 lane out of the 4 to the output */ + data = meson_codec_glue_input_get_data(dai); + data->params.channels_min = min_t(unsigned int, AIU_ACODEC_OUT_CHMAX, + data->params.channels_min); + data->params.channels_max = min_t(unsigned int, AIU_ACODEC_OUT_CHMAX, + data->params.channels_max); + + return 0; +} + +static const struct snd_soc_dai_ops aiu_acodec_ctrl_input_ops = { + .hw_params = aiu_acodec_ctrl_input_hw_params, + .set_fmt = meson_codec_glue_input_set_fmt, +}; + +static const struct snd_soc_dai_ops aiu_acodec_ctrl_output_ops = { + .startup = meson_codec_glue_output_startup, +}; + +#define AIU_ACODEC_CTRL_FORMATS \ + (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE | \ + SNDRV_PCM_FMTBIT_S24_3LE | SNDRV_PCM_FMTBIT_S24_LE | \ + SNDRV_PCM_FMTBIT_S32_LE) + +#define AIU_ACODEC_STREAM(xname, xsuffix, xchmax) \ +{ \ + .stream_name = xname " " xsuffix, \ + .channels_min = 1, \ + .channels_max = (xchmax), \ + .rate_min = 5512, \ + .rate_max = 192000, \ + .formats = AIU_ACODEC_CTRL_FORMATS, \ +} + +#define AIU_ACODEC_INPUT(xname) { \ + .name = "ACODEC CTRL " xname, \ + .playback = AIU_ACODEC_STREAM(xname, "Playback", 8), \ + .ops = &aiu_acodec_ctrl_input_ops, \ + .probe = meson_codec_glue_input_dai_probe, \ + .remove = meson_codec_glue_input_dai_remove, \ +} + +#define AIU_ACODEC_OUTPUT(xname) { \ + .name = "ACODEC CTRL " xname, \ + .capture = AIU_ACODEC_STREAM(xname, "Capture", AIU_ACODEC_OUT_CHMAX), \ + .ops = &aiu_acodec_ctrl_output_ops, \ +} + +static struct snd_soc_dai_driver aiu_acodec_ctrl_dai_drv[] = { + [CTRL_I2S] = AIU_ACODEC_INPUT("ACODEC I2S IN"), + [CTRL_PCM] = AIU_ACODEC_INPUT("ACODEC PCM IN"), + [CTRL_OUT] = AIU_ACODEC_OUTPUT("ACODEC OUT"), +}; + +static const struct snd_soc_dapm_route aiu_acodec_ctrl_routes[] = { + { "ACODEC SRC", "I2S", "ACODEC I2S IN Playback" }, + { "ACODEC SRC", "PCM", "ACODEC PCM IN Playback" }, + { "ACODEC OUT EN", "Switch", "ACODEC SRC" }, + { "ACODEC OUT Capture", NULL, "ACODEC OUT EN" }, +}; + +static const struct snd_kcontrol_new aiu_acodec_ctrl_controls[] = { + SOC_SINGLE("ACODEC I2S Lane Select", AIU_ACODEC_CTRL, + CTRL_I2S_OUT_LANE_SRC, 3, 0), +}; + +static int aiu_acodec_of_xlate_dai_name(struct snd_soc_component *component, + struct of_phandle_args *args, + const char **dai_name) +{ + return aiu_of_xlate_dai_name(component, args, dai_name, AIU_ACODEC); +} + +static int aiu_acodec_ctrl_component_probe(struct snd_soc_component *component) +{ + /* + * NOTE: Din Skew setting + * According to the documentation, the following update adds one delay + * to the din line. Without this, the output saturates. This happens + * regardless of the link format (i2s or left_j) so it is not clear what + * it actually does but it seems to be required + */ + snd_soc_component_update_bits(component, AIU_ACODEC_CTRL, + CTRL_DIN_SKEW, + FIELD_PREP(CTRL_DIN_SKEW, 2)); + + return 0; +} + +static const struct snd_soc_component_driver aiu_acodec_ctrl_component = { + .name = "AIU Internal DAC Codec Control", + .probe = aiu_acodec_ctrl_component_probe, + .controls = aiu_acodec_ctrl_controls, + .num_controls = ARRAY_SIZE(aiu_acodec_ctrl_controls), + .dapm_widgets = aiu_acodec_ctrl_widgets, + .num_dapm_widgets = ARRAY_SIZE(aiu_acodec_ctrl_widgets), + .dapm_routes = aiu_acodec_ctrl_routes, + .num_dapm_routes = ARRAY_SIZE(aiu_acodec_ctrl_routes), + .of_xlate_dai_name = aiu_acodec_of_xlate_dai_name, + .endianness = 1, + .non_legacy_dai_naming = 1, +}; + +int aiu_acodec_ctrl_register_component(struct device *dev) +{ + return snd_soc_register_component(dev, &aiu_acodec_ctrl_component, + aiu_acodec_ctrl_dai_drv, + ARRAY_SIZE(aiu_acodec_ctrl_dai_drv)); +} diff --git a/sound/soc/meson/aiu-codec-ctrl.c b/sound/soc/meson/aiu-codec-ctrl.c new file mode 100644 index 000000000000..4b773d3e8b07 --- /dev/null +++ b/sound/soc/meson/aiu-codec-ctrl.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include + +#include +#include "aiu.h" +#include "meson-codec-glue.h" + +#define CTRL_CLK_SEL GENMASK(1, 0) +#define CTRL_DATA_SEL_SHIFT 4 +#define CTRL_DATA_SEL (0x3 << CTRL_DATA_SEL_SHIFT) + +static const char * const aiu_codec_ctrl_mux_texts[] = { + "DISABLED", "PCM", "I2S", +}; + +static int aiu_codec_ctrl_mux_put_enum(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = + snd_soc_dapm_kcontrol_component(kcontrol); + struct snd_soc_dapm_context *dapm = + snd_soc_dapm_kcontrol_dapm(kcontrol); + struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; + unsigned int mux, changed; + + mux = snd_soc_enum_item_to_val(e, ucontrol->value.enumerated.item[0]); + changed = snd_soc_component_test_bits(component, e->reg, + CTRL_DATA_SEL, + FIELD_PREP(CTRL_DATA_SEL, mux)); + + if (!changed) + return 0; + + /* Force disconnect of the mux while updating */ + snd_soc_dapm_mux_update_power(dapm, kcontrol, 0, NULL, NULL); + + /* Reset the source first */ + snd_soc_component_update_bits(component, e->reg, + CTRL_CLK_SEL | + CTRL_DATA_SEL, + FIELD_PREP(CTRL_CLK_SEL, 0) | + FIELD_PREP(CTRL_DATA_SEL, 0)); + + /* Set the appropriate source */ + snd_soc_component_update_bits(component, e->reg, + CTRL_CLK_SEL | + CTRL_DATA_SEL, + FIELD_PREP(CTRL_CLK_SEL, mux) | + FIELD_PREP(CTRL_DATA_SEL, mux)); + + snd_soc_dapm_mux_update_power(dapm, kcontrol, mux, e, NULL); + + return 0; +} + +static SOC_ENUM_SINGLE_DECL(aiu_hdmi_ctrl_mux_enum, AIU_HDMI_CLK_DATA_CTRL, + CTRL_DATA_SEL_SHIFT, + aiu_codec_ctrl_mux_texts); + +static const struct snd_kcontrol_new aiu_hdmi_ctrl_mux = + SOC_DAPM_ENUM_EXT("HDMI Source", aiu_hdmi_ctrl_mux_enum, + snd_soc_dapm_get_enum_double, + aiu_codec_ctrl_mux_put_enum); + +static const struct snd_soc_dapm_widget aiu_hdmi_ctrl_widgets[] = { + SND_SOC_DAPM_MUX("HDMI CTRL SRC", SND_SOC_NOPM, 0, 0, + &aiu_hdmi_ctrl_mux), +}; + +static const struct snd_soc_dai_ops aiu_codec_ctrl_input_ops = { + .hw_params = meson_codec_glue_input_hw_params, + .set_fmt = meson_codec_glue_input_set_fmt, +}; + +static const struct snd_soc_dai_ops aiu_codec_ctrl_output_ops = { + .startup = meson_codec_glue_output_startup, +}; + +#define AIU_CODEC_CTRL_FORMATS \ + (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE | \ + SNDRV_PCM_FMTBIT_S24_3LE | SNDRV_PCM_FMTBIT_S24_LE | \ + SNDRV_PCM_FMTBIT_S32_LE) + +#define AIU_CODEC_CTRL_STREAM(xname, xsuffix) \ +{ \ + .stream_name = xname " " xsuffix, \ + .channels_min = 1, \ + .channels_max = 8, \ + .rate_min = 5512, \ + .rate_max = 192000, \ + .formats = AIU_CODEC_CTRL_FORMATS, \ +} + +#define AIU_CODEC_CTRL_INPUT(xname) { \ + .name = "CODEC CTRL " xname, \ + .playback = AIU_CODEC_CTRL_STREAM(xname, "Playback"), \ + .ops = &aiu_codec_ctrl_input_ops, \ + .probe = meson_codec_glue_input_dai_probe, \ + .remove = meson_codec_glue_input_dai_remove, \ +} + +#define AIU_CODEC_CTRL_OUTPUT(xname) { \ + .name = "CODEC CTRL " xname, \ + .capture = AIU_CODEC_CTRL_STREAM(xname, "Capture"), \ + .ops = &aiu_codec_ctrl_output_ops, \ +} + +static struct snd_soc_dai_driver aiu_hdmi_ctrl_dai_drv[] = { + [CTRL_I2S] = AIU_CODEC_CTRL_INPUT("HDMI I2S IN"), + [CTRL_PCM] = AIU_CODEC_CTRL_INPUT("HDMI PCM IN"), + [CTRL_OUT] = AIU_CODEC_CTRL_OUTPUT("HDMI OUT"), +}; + +static const struct snd_soc_dapm_route aiu_hdmi_ctrl_routes[] = { + { "HDMI CTRL SRC", "I2S", "HDMI I2S IN Playback" }, + { "HDMI CTRL SRC", "PCM", "HDMI PCM IN Playback" }, + { "HDMI OUT Capture", NULL, "HDMI CTRL SRC" }, +}; + +static int aiu_hdmi_of_xlate_dai_name(struct snd_soc_component *component, + struct of_phandle_args *args, + const char **dai_name) +{ + return aiu_of_xlate_dai_name(component, args, dai_name, AIU_HDMI); +} + +static const struct snd_soc_component_driver aiu_hdmi_ctrl_component = { + .name = "AIU HDMI Codec Control", + .dapm_widgets = aiu_hdmi_ctrl_widgets, + .num_dapm_widgets = ARRAY_SIZE(aiu_hdmi_ctrl_widgets), + .dapm_routes = aiu_hdmi_ctrl_routes, + .num_dapm_routes = ARRAY_SIZE(aiu_hdmi_ctrl_routes), + .of_xlate_dai_name = aiu_hdmi_of_xlate_dai_name, + .endianness = 1, + .non_legacy_dai_naming = 1, +}; + +int aiu_hdmi_ctrl_register_component(struct device *dev) +{ + return snd_soc_register_component(dev, &aiu_hdmi_ctrl_component, + aiu_hdmi_ctrl_dai_drv, + ARRAY_SIZE(aiu_hdmi_ctrl_dai_drv)); +} + diff --git a/sound/soc/meson/aiu-encoder-i2s.c b/sound/soc/meson/aiu-encoder-i2s.c new file mode 100644 index 000000000000..832e22d275fe --- /dev/null +++ b/sound/soc/meson/aiu-encoder-i2s.c @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include +#include + +#include "aiu.h" + +#define AIU_I2S_SOURCE_DESC_MODE_8CH BIT(0) +#define AIU_I2S_SOURCE_DESC_MODE_24BIT BIT(5) +#define AIU_I2S_SOURCE_DESC_MODE_32BIT BIT(9) +#define AIU_I2S_SOURCE_DESC_MODE_SPLIT BIT(11) +#define AIU_RST_SOFT_I2S_FAST BIT(0) + +#define AIU_I2S_DAC_CFG_MSB_FIRST BIT(2) +#define AIU_I2S_MISC_HOLD_EN BIT(2) +#define AIU_CLK_CTRL_I2S_DIV_EN BIT(0) +#define AIU_CLK_CTRL_I2S_DIV GENMASK(3, 2) +#define AIU_CLK_CTRL_AOCLK_INVERT BIT(6) +#define AIU_CLK_CTRL_LRCLK_INVERT BIT(7) +#define AIU_CLK_CTRL_LRCLK_SKEW GENMASK(9, 8) +#define AIU_CLK_CTRL_MORE_HDMI_AMCLK BIT(6) +#define AIU_CLK_CTRL_MORE_I2S_DIV GENMASK(5, 0) +#define AIU_CODEC_DAC_LRCLK_CTRL_DIV GENMASK(11, 0) + +static void aiu_encoder_i2s_divider_enable(struct snd_soc_component *component, + bool enable) +{ + snd_soc_component_update_bits(component, AIU_CLK_CTRL, + AIU_CLK_CTRL_I2S_DIV_EN, + enable ? AIU_CLK_CTRL_I2S_DIV_EN : 0); +} + +static void aiu_encoder_i2s_hold(struct snd_soc_component *component, + bool enable) +{ + snd_soc_component_update_bits(component, AIU_I2S_MISC, + AIU_I2S_MISC_HOLD_EN, + enable ? AIU_I2S_MISC_HOLD_EN : 0); +} + +static int aiu_encoder_i2s_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + aiu_encoder_i2s_hold(component, false); + return 0; + + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + aiu_encoder_i2s_hold(component, true); + return 0; + + default: + return -EINVAL; + } +} + +static int aiu_encoder_i2s_setup_desc(struct snd_soc_component *component, + struct snd_pcm_hw_params *params) +{ + /* Always operate in split (classic interleaved) mode */ + unsigned int desc = AIU_I2S_SOURCE_DESC_MODE_SPLIT; + unsigned int val; + + /* Reset required to update the pipeline */ + snd_soc_component_write(component, AIU_RST_SOFT, AIU_RST_SOFT_I2S_FAST); + snd_soc_component_read(component, AIU_I2S_SYNC, &val); + + switch (params_physical_width(params)) { + case 16: /* Nothing to do */ + break; + + case 32: + desc |= (AIU_I2S_SOURCE_DESC_MODE_24BIT | + AIU_I2S_SOURCE_DESC_MODE_32BIT); + break; + + default: + return -EINVAL; + } + + switch (params_channels(params)) { + case 2: /* Nothing to do */ + break; + case 8: + desc |= AIU_I2S_SOURCE_DESC_MODE_8CH; + break; + default: + return -EINVAL; + } + + snd_soc_component_update_bits(component, AIU_I2S_SOURCE_DESC, + AIU_I2S_SOURCE_DESC_MODE_8CH | + AIU_I2S_SOURCE_DESC_MODE_24BIT | + AIU_I2S_SOURCE_DESC_MODE_32BIT | + AIU_I2S_SOURCE_DESC_MODE_SPLIT, + desc); + + return 0; +} + +static int aiu_encoder_i2s_set_legacy_div(struct snd_soc_component *component, + struct snd_pcm_hw_params *params, + unsigned int bs) +{ + switch (bs) { + case 1: + case 2: + case 4: + case 8: + /* These are the only valid legacy dividers */ + break; + + default: + dev_err(component->dev, "Unsupported i2s divider: %u\n", bs); + return -EINVAL; + } + + snd_soc_component_update_bits(component, AIU_CLK_CTRL, + AIU_CLK_CTRL_I2S_DIV, + FIELD_PREP(AIU_CLK_CTRL_I2S_DIV, + __ffs(bs))); + + snd_soc_component_update_bits(component, AIU_CLK_CTRL_MORE, + AIU_CLK_CTRL_MORE_I2S_DIV, + FIELD_PREP(AIU_CLK_CTRL_MORE_I2S_DIV, + 0)); + + return 0; +} + +static int aiu_encoder_i2s_set_more_div(struct snd_soc_component *component, + struct snd_pcm_hw_params *params, + unsigned int bs) +{ + /* + * NOTE: this HW is odd. + * In most configuration, the i2s divider is 'mclk / blck'. + * However, in 16 bits - 8ch mode, this factor needs to be + * increased by 50% to get the correct output rate. + * No idea why ! + */ + if (params_width(params) == 16 && params_channels(params) == 8) { + if (bs % 2) { + dev_err(component->dev, + "Cannot increase i2s divider by 50%%\n"); + return -EINVAL; + } + bs += bs / 2; + } + + /* Use CLK_MORE for mclk to bclk divider */ + snd_soc_component_update_bits(component, AIU_CLK_CTRL, + AIU_CLK_CTRL_I2S_DIV, + FIELD_PREP(AIU_CLK_CTRL_I2S_DIV, 0)); + + snd_soc_component_update_bits(component, AIU_CLK_CTRL_MORE, + AIU_CLK_CTRL_MORE_I2S_DIV, + FIELD_PREP(AIU_CLK_CTRL_MORE_I2S_DIV, + bs - 1)); + + return 0; +} + +static int aiu_encoder_i2s_set_clocks(struct snd_soc_component *component, + struct snd_pcm_hw_params *params) +{ + struct aiu *aiu = snd_soc_component_get_drvdata(component); + unsigned int srate = params_rate(params); + unsigned int fs, bs; + int ret; + + /* Get the oversampling factor */ + fs = DIV_ROUND_CLOSEST(clk_get_rate(aiu->i2s.clks[MCLK].clk), srate); + + if (fs % 64) + return -EINVAL; + + /* Send data MSB first */ + snd_soc_component_update_bits(component, AIU_I2S_DAC_CFG, + AIU_I2S_DAC_CFG_MSB_FIRST, + AIU_I2S_DAC_CFG_MSB_FIRST); + + /* Set bclk to lrlck ratio */ + snd_soc_component_update_bits(component, AIU_CODEC_DAC_LRCLK_CTRL, + AIU_CODEC_DAC_LRCLK_CTRL_DIV, + FIELD_PREP(AIU_CODEC_DAC_LRCLK_CTRL_DIV, + 64 - 1)); + + bs = fs / 64; + + if (aiu->platform->has_clk_ctrl_more_i2s_div) + ret = aiu_encoder_i2s_set_more_div(component, params, bs); + else + ret = aiu_encoder_i2s_set_legacy_div(component, params, bs); + + if (ret) + return ret; + + /* Make sure amclk is used for HDMI i2s as well */ + snd_soc_component_update_bits(component, AIU_CLK_CTRL_MORE, + AIU_CLK_CTRL_MORE_HDMI_AMCLK, + AIU_CLK_CTRL_MORE_HDMI_AMCLK); + + return 0; +} + +static int aiu_encoder_i2s_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + int ret; + + /* Disable the clock while changing the settings */ + aiu_encoder_i2s_divider_enable(component, false); + + ret = aiu_encoder_i2s_setup_desc(component, params); + if (ret) { + dev_err(dai->dev, "setting i2s desc failed\n"); + return ret; + } + + ret = aiu_encoder_i2s_set_clocks(component, params); + if (ret) { + dev_err(dai->dev, "setting i2s clocks failed\n"); + return ret; + } + + aiu_encoder_i2s_divider_enable(component, true); + + return 0; +} + +static int aiu_encoder_i2s_hw_free(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + + aiu_encoder_i2s_divider_enable(component, false); + + return 0; +} + +static int aiu_encoder_i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) +{ + struct snd_soc_component *component = dai->component; + unsigned int inv = fmt & SND_SOC_DAIFMT_INV_MASK; + unsigned int val = 0; + unsigned int skew; + + /* Only CPU Master / Codec Slave supported ATM */ + if ((fmt & SND_SOC_DAIFMT_MASTER_MASK) != SND_SOC_DAIFMT_CBS_CFS) + return -EINVAL; + + if (inv == SND_SOC_DAIFMT_NB_IF || + inv == SND_SOC_DAIFMT_IB_IF) + val |= AIU_CLK_CTRL_LRCLK_INVERT; + + if (inv == SND_SOC_DAIFMT_IB_NF || + inv == SND_SOC_DAIFMT_IB_IF) + val |= AIU_CLK_CTRL_AOCLK_INVERT; + + /* Signal skew */ + switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { + case SND_SOC_DAIFMT_I2S: + /* Invert sample clock for i2s */ + val ^= AIU_CLK_CTRL_LRCLK_INVERT; + skew = 1; + break; + case SND_SOC_DAIFMT_LEFT_J: + skew = 0; + break; + default: + return -EINVAL; + } + + val |= FIELD_PREP(AIU_CLK_CTRL_LRCLK_SKEW, skew); + snd_soc_component_update_bits(component, AIU_CLK_CTRL, + AIU_CLK_CTRL_LRCLK_INVERT | + AIU_CLK_CTRL_AOCLK_INVERT | + AIU_CLK_CTRL_LRCLK_SKEW, + val); + + return 0; +} + +static int aiu_encoder_i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, + unsigned int freq, int dir) +{ + struct aiu *aiu = snd_soc_component_get_drvdata(dai->component); + int ret; + + if (WARN_ON(clk_id != 0)) + return -EINVAL; + + if (dir == SND_SOC_CLOCK_IN) + return 0; + + ret = clk_set_rate(aiu->i2s.clks[MCLK].clk, freq); + if (ret) + dev_err(dai->dev, "Failed to set sysclk to %uHz", freq); + + return ret; +} + +static const unsigned int hw_channels[] = {2, 8}; +static const struct snd_pcm_hw_constraint_list hw_channel_constraints = { + .list = hw_channels, + .count = ARRAY_SIZE(hw_channels), + .mask = 0, +}; + +static int aiu_encoder_i2s_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct aiu *aiu = snd_soc_component_get_drvdata(dai->component); + int ret; + + /* Make sure the encoder gets either 2 or 8 channels */ + ret = snd_pcm_hw_constraint_list(substream->runtime, 0, + SNDRV_PCM_HW_PARAM_CHANNELS, + &hw_channel_constraints); + if (ret) { + dev_err(dai->dev, "adding channels constraints failed\n"); + return ret; + } + + ret = clk_bulk_prepare_enable(aiu->i2s.clk_num, aiu->i2s.clks); + if (ret) + dev_err(dai->dev, "failed to enable i2s clocks\n"); + + return ret; +} + +static void aiu_encoder_i2s_shutdown(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct aiu *aiu = snd_soc_component_get_drvdata(dai->component); + + clk_bulk_disable_unprepare(aiu->i2s.clk_num, aiu->i2s.clks); +} + +const struct snd_soc_dai_ops aiu_encoder_i2s_dai_ops = { + .trigger = aiu_encoder_i2s_trigger, + .hw_params = aiu_encoder_i2s_hw_params, + .hw_free = aiu_encoder_i2s_hw_free, + .set_fmt = aiu_encoder_i2s_set_fmt, + .set_sysclk = aiu_encoder_i2s_set_sysclk, + .startup = aiu_encoder_i2s_startup, + .shutdown = aiu_encoder_i2s_shutdown, +}; + diff --git a/sound/soc/meson/aiu-encoder-spdif.c b/sound/soc/meson/aiu-encoder-spdif.c new file mode 100644 index 000000000000..de850913975f --- /dev/null +++ b/sound/soc/meson/aiu-encoder-spdif.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include +#include +#include + +#include "aiu.h" + +#define AIU_958_MISC_NON_PCM BIT(0) +#define AIU_958_MISC_MODE_16BITS BIT(1) +#define AIU_958_MISC_16BITS_ALIGN GENMASK(6, 5) +#define AIU_958_MISC_MODE_32BITS BIT(7) +#define AIU_958_MISC_U_FROM_STREAM BIT(12) +#define AIU_958_MISC_FORCE_LR BIT(13) +#define AIU_958_CTRL_HOLD_EN BIT(0) +#define AIU_CLK_CTRL_958_DIV_EN BIT(1) +#define AIU_CLK_CTRL_958_DIV GENMASK(5, 4) +#define AIU_CLK_CTRL_958_DIV_MORE BIT(12) + +#define AIU_CS_WORD_LEN 4 +#define AIU_958_INTERNAL_DIV 2 + +static void +aiu_encoder_spdif_divider_enable(struct snd_soc_component *component, + bool enable) +{ + snd_soc_component_update_bits(component, AIU_CLK_CTRL, + AIU_CLK_CTRL_958_DIV_EN, + enable ? AIU_CLK_CTRL_958_DIV_EN : 0); +} + +static void aiu_encoder_spdif_hold(struct snd_soc_component *component, + bool enable) +{ + snd_soc_component_update_bits(component, AIU_958_CTRL, + AIU_958_CTRL_HOLD_EN, + enable ? AIU_958_CTRL_HOLD_EN : 0); +} + +static int +aiu_encoder_spdif_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + aiu_encoder_spdif_hold(component, false); + return 0; + + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + aiu_encoder_spdif_hold(component, true); + return 0; + + default: + return -EINVAL; + } +} + +static int aiu_encoder_spdif_setup_cs_word(struct snd_soc_component *component, + struct snd_pcm_hw_params *params) +{ + u8 cs[AIU_CS_WORD_LEN]; + unsigned int val; + int ret; + + ret = snd_pcm_create_iec958_consumer_hw_params(params, cs, + AIU_CS_WORD_LEN); + if (ret < 0) + return ret; + + /* Write the 1st half word */ + val = cs[1] | cs[0] << 8; + snd_soc_component_write(component, AIU_958_CHSTAT_L0, val); + snd_soc_component_write(component, AIU_958_CHSTAT_R0, val); + + /* Write the 2nd half word */ + val = cs[3] | cs[2] << 8; + snd_soc_component_write(component, AIU_958_CHSTAT_L1, val); + snd_soc_component_write(component, AIU_958_CHSTAT_R1, val); + + return 0; +} + +static int aiu_encoder_spdif_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct aiu *aiu = snd_soc_component_get_drvdata(component); + unsigned int val = 0, mrate; + int ret; + + /* Disable the clock while changing the settings */ + aiu_encoder_spdif_divider_enable(component, false); + + switch (params_physical_width(params)) { + case 16: + val |= AIU_958_MISC_MODE_16BITS; + val |= FIELD_PREP(AIU_958_MISC_16BITS_ALIGN, 2); + break; + case 32: + val |= AIU_958_MISC_MODE_32BITS; + break; + default: + dev_err(dai->dev, "Unsupport physical width\n"); + return -EINVAL; + } + + snd_soc_component_update_bits(component, AIU_958_MISC, + AIU_958_MISC_NON_PCM | + AIU_958_MISC_MODE_16BITS | + AIU_958_MISC_16BITS_ALIGN | + AIU_958_MISC_MODE_32BITS | + AIU_958_MISC_FORCE_LR | + AIU_958_MISC_U_FROM_STREAM, + val); + + /* Set the stream channel status word */ + ret = aiu_encoder_spdif_setup_cs_word(component, params); + if (ret) { + dev_err(dai->dev, "failed to set channel status word\n"); + return ret; + } + + snd_soc_component_update_bits(component, AIU_CLK_CTRL, + AIU_CLK_CTRL_958_DIV | + AIU_CLK_CTRL_958_DIV_MORE, + FIELD_PREP(AIU_CLK_CTRL_958_DIV, + __ffs(AIU_958_INTERNAL_DIV))); + + /* 2 * 32bits per subframe * 2 channels = 128 */ + mrate = params_rate(params) * 128 * AIU_958_INTERNAL_DIV; + ret = clk_set_rate(aiu->spdif.clks[MCLK].clk, mrate); + if (ret) { + dev_err(dai->dev, "failed to set mclk rate\n"); + return ret; + } + + aiu_encoder_spdif_divider_enable(component, true); + + return 0; +} + +static int aiu_encoder_spdif_hw_free(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + + aiu_encoder_spdif_divider_enable(component, false); + + return 0; +} + +static int aiu_encoder_spdif_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct aiu *aiu = snd_soc_component_get_drvdata(dai->component); + int ret; + + /* + * NOTE: Make sure the spdif block is on its own divider. + * + * The spdif can be clocked by the i2s master clock or its own + * clock. We should (in theory) change the source depending on the + * origin of the data. + * + * However, considering the clocking scheme used on these platforms, + * the master clocks will pick the same PLL source when they are + * playing from the same FIFO. The clock should be in sync so, it + * should not be necessary to reparent the spdif master clock. + */ + ret = clk_set_parent(aiu->spdif.clks[MCLK].clk, + aiu->spdif_mclk); + if (ret) + return ret; + + ret = clk_bulk_prepare_enable(aiu->spdif.clk_num, aiu->spdif.clks); + if (ret) + dev_err(dai->dev, "failed to enable spdif clocks\n"); + + return ret; +} + +static void aiu_encoder_spdif_shutdown(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct aiu *aiu = snd_soc_component_get_drvdata(dai->component); + + clk_bulk_disable_unprepare(aiu->spdif.clk_num, aiu->spdif.clks); +} + +const struct snd_soc_dai_ops aiu_encoder_spdif_dai_ops = { + .trigger = aiu_encoder_spdif_trigger, + .hw_params = aiu_encoder_spdif_hw_params, + .hw_free = aiu_encoder_spdif_hw_free, + .startup = aiu_encoder_spdif_startup, + .shutdown = aiu_encoder_spdif_shutdown, +}; diff --git a/sound/soc/meson/aiu-fifo-i2s.c b/sound/soc/meson/aiu-fifo-i2s.c new file mode 100644 index 000000000000..9a5271ce80fe --- /dev/null +++ b/sound/soc/meson/aiu-fifo-i2s.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include +#include + +#include "aiu.h" +#include "aiu-fifo.h" + +#define AIU_I2S_SOURCE_DESC_MODE_8CH BIT(0) +#define AIU_I2S_SOURCE_DESC_MODE_24BIT BIT(5) +#define AIU_I2S_SOURCE_DESC_MODE_32BIT BIT(9) +#define AIU_I2S_SOURCE_DESC_MODE_SPLIT BIT(11) +#define AIU_MEM_I2S_MASKS_IRQ_BLOCK GENMASK(31, 16) +#define AIU_MEM_I2S_CONTROL_MODE_16BIT BIT(6) +#define AIU_MEM_I2S_BUF_CNTL_INIT BIT(0) +#define AIU_RST_SOFT_I2S_FAST BIT(0) + +#define AIU_FIFO_I2S_BLOCK 256 + +static struct snd_pcm_hardware fifo_i2s_pcm = { + .info = (SNDRV_PCM_INFO_INTERLEAVED | + SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_PAUSE), + .formats = AIU_FORMATS, + .rate_min = 5512, + .rate_max = 192000, + .channels_min = 2, + .channels_max = 8, + .period_bytes_min = AIU_FIFO_I2S_BLOCK, + .period_bytes_max = AIU_FIFO_I2S_BLOCK * USHRT_MAX, + .periods_min = 2, + .periods_max = UINT_MAX, + + /* No real justification for this */ + .buffer_bytes_max = 1 * 1024 * 1024, +}; + +static int aiu_fifo_i2s_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + unsigned int val; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + snd_soc_component_write(component, AIU_RST_SOFT, + AIU_RST_SOFT_I2S_FAST); + snd_soc_component_read(component, AIU_I2S_SYNC, &val); + break; + } + + return aiu_fifo_trigger(substream, cmd, dai); +} + +static int aiu_fifo_i2s_prepare(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + int ret; + + ret = aiu_fifo_prepare(substream, dai); + if (ret) + return ret; + + snd_soc_component_update_bits(component, + AIU_MEM_I2S_BUF_CNTL, + AIU_MEM_I2S_BUF_CNTL_INIT, + AIU_MEM_I2S_BUF_CNTL_INIT); + snd_soc_component_update_bits(component, + AIU_MEM_I2S_BUF_CNTL, + AIU_MEM_I2S_BUF_CNTL_INIT, 0); + + return 0; +} + +static int aiu_fifo_i2s_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct aiu_fifo *fifo = dai->playback_dma_data; + unsigned int val; + int ret; + + ret = aiu_fifo_hw_params(substream, params, dai); + if (ret) + return ret; + + switch (params_physical_width(params)) { + case 16: + val = AIU_MEM_I2S_CONTROL_MODE_16BIT; + break; + case 32: + val = 0; + break; + default: + dev_err(dai->dev, "Unsupported physical width %u\n", + params_physical_width(params)); + return -EINVAL; + } + + snd_soc_component_update_bits(component, AIU_MEM_I2S_CONTROL, + AIU_MEM_I2S_CONTROL_MODE_16BIT, + val); + + /* Setup the irq periodicity */ + val = params_period_bytes(params) / fifo->fifo_block; + val = FIELD_PREP(AIU_MEM_I2S_MASKS_IRQ_BLOCK, val); + snd_soc_component_update_bits(component, AIU_MEM_I2S_MASKS, + AIU_MEM_I2S_MASKS_IRQ_BLOCK, val); + + return 0; +} + +const struct snd_soc_dai_ops aiu_fifo_i2s_dai_ops = { + .trigger = aiu_fifo_i2s_trigger, + .prepare = aiu_fifo_i2s_prepare, + .hw_params = aiu_fifo_i2s_hw_params, + .hw_free = aiu_fifo_hw_free, + .startup = aiu_fifo_startup, + .shutdown = aiu_fifo_shutdown, +}; + +int aiu_fifo_i2s_dai_probe(struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct aiu *aiu = snd_soc_component_get_drvdata(component); + struct aiu_fifo *fifo; + int ret; + + ret = aiu_fifo_dai_probe(dai); + if (ret) + return ret; + + fifo = dai->playback_dma_data; + + fifo->pcm = &fifo_i2s_pcm; + fifo->mem_offset = AIU_MEM_I2S_START; + fifo->fifo_block = AIU_FIFO_I2S_BLOCK; + fifo->pclk = aiu->i2s.clks[PCLK].clk; + fifo->irq = aiu->i2s.irq; + + return 0; +} diff --git a/sound/soc/meson/aiu-fifo-spdif.c b/sound/soc/meson/aiu-fifo-spdif.c new file mode 100644 index 000000000000..44eb6faacf44 --- /dev/null +++ b/sound/soc/meson/aiu-fifo-spdif.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include + +#include "aiu.h" +#include "aiu-fifo.h" + +#define AIU_IEC958_DCU_FF_CTRL_EN BIT(0) +#define AIU_IEC958_DCU_FF_CTRL_AUTO_DISABLE BIT(1) +#define AIU_IEC958_DCU_FF_CTRL_IRQ_MODE GENMASK(3, 2) +#define AIU_IEC958_DCU_FF_CTRL_IRQ_OUT_THD BIT(2) +#define AIU_IEC958_DCU_FF_CTRL_IRQ_FRAME_READ BIT(3) +#define AIU_IEC958_DCU_FF_CTRL_SYNC_HEAD_EN BIT(4) +#define AIU_IEC958_DCU_FF_CTRL_BYTE_SEEK BIT(5) +#define AIU_IEC958_DCU_FF_CTRL_CONTINUE BIT(6) +#define AIU_MEM_IEC958_CONTROL_ENDIAN GENMASK(5, 3) +#define AIU_MEM_IEC958_CONTROL_RD_DDR BIT(6) +#define AIU_MEM_IEC958_CONTROL_MODE_16BIT BIT(7) +#define AIU_MEM_IEC958_CONTROL_MODE_LINEAR BIT(8) +#define AIU_MEM_IEC958_BUF_CNTL_INIT BIT(0) + +#define AIU_FIFO_SPDIF_BLOCK 8 + +static struct snd_pcm_hardware fifo_spdif_pcm = { + .info = (SNDRV_PCM_INFO_INTERLEAVED | + SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_PAUSE), + .formats = AIU_FORMATS, + .rate_min = 5512, + .rate_max = 192000, + .channels_min = 2, + .channels_max = 2, + .period_bytes_min = AIU_FIFO_SPDIF_BLOCK, + .period_bytes_max = AIU_FIFO_SPDIF_BLOCK * USHRT_MAX, + .periods_min = 2, + .periods_max = UINT_MAX, + + /* No real justification for this */ + .buffer_bytes_max = 1 * 1024 * 1024, +}; + +static void fifo_spdif_dcu_enable(struct snd_soc_component *component, + bool enable) +{ + snd_soc_component_update_bits(component, AIU_IEC958_DCU_FF_CTRL, + AIU_IEC958_DCU_FF_CTRL_EN, + enable ? AIU_IEC958_DCU_FF_CTRL_EN : 0); +} + +static int fifo_spdif_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + int ret; + + ret = aiu_fifo_trigger(substream, cmd, dai); + if (ret) + return ret; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + fifo_spdif_dcu_enable(component, true); + break; + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + case SNDRV_PCM_TRIGGER_STOP: + fifo_spdif_dcu_enable(component, false); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int fifo_spdif_prepare(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + int ret; + + ret = aiu_fifo_prepare(substream, dai); + if (ret) + return ret; + + snd_soc_component_update_bits(component, + AIU_MEM_IEC958_BUF_CNTL, + AIU_MEM_IEC958_BUF_CNTL_INIT, + AIU_MEM_IEC958_BUF_CNTL_INIT); + snd_soc_component_update_bits(component, + AIU_MEM_IEC958_BUF_CNTL, + AIU_MEM_IEC958_BUF_CNTL_INIT, 0); + + return 0; +} + +static int fifo_spdif_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + unsigned int val; + int ret; + + ret = aiu_fifo_hw_params(substream, params, dai); + if (ret) + return ret; + + val = AIU_MEM_IEC958_CONTROL_RD_DDR | + AIU_MEM_IEC958_CONTROL_MODE_LINEAR; + + switch (params_physical_width(params)) { + case 16: + val |= AIU_MEM_IEC958_CONTROL_MODE_16BIT; + break; + case 32: + break; + default: + dev_err(dai->dev, "Unsupported physical width %u\n", + params_physical_width(params)); + return -EINVAL; + } + + snd_soc_component_update_bits(component, AIU_MEM_IEC958_CONTROL, + AIU_MEM_IEC958_CONTROL_ENDIAN | + AIU_MEM_IEC958_CONTROL_RD_DDR | + AIU_MEM_IEC958_CONTROL_MODE_LINEAR | + AIU_MEM_IEC958_CONTROL_MODE_16BIT, + val); + + /* Number bytes read by the FIFO between each IRQ */ + snd_soc_component_write(component, AIU_IEC958_BPF, + params_period_bytes(params)); + + /* + * AUTO_DISABLE and SYNC_HEAD are enabled by default but + * this should be disabled in PCM (uncompressed) mode + */ + snd_soc_component_update_bits(component, AIU_IEC958_DCU_FF_CTRL, + AIU_IEC958_DCU_FF_CTRL_AUTO_DISABLE | + AIU_IEC958_DCU_FF_CTRL_IRQ_MODE | + AIU_IEC958_DCU_FF_CTRL_SYNC_HEAD_EN, + AIU_IEC958_DCU_FF_CTRL_IRQ_FRAME_READ); + + return 0; +} + +const struct snd_soc_dai_ops aiu_fifo_spdif_dai_ops = { + .trigger = fifo_spdif_trigger, + .prepare = fifo_spdif_prepare, + .hw_params = fifo_spdif_hw_params, + .hw_free = aiu_fifo_hw_free, + .startup = aiu_fifo_startup, + .shutdown = aiu_fifo_shutdown, +}; + +int aiu_fifo_spdif_dai_probe(struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct aiu *aiu = snd_soc_component_get_drvdata(component); + struct aiu_fifo *fifo; + int ret; + + ret = aiu_fifo_dai_probe(dai); + if (ret) + return ret; + + fifo = dai->playback_dma_data; + + fifo->pcm = &fifo_spdif_pcm; + fifo->mem_offset = AIU_MEM_IEC958_START; + fifo->fifo_block = 1; + fifo->pclk = aiu->spdif.clks[PCLK].clk; + fifo->irq = aiu->spdif.irq; + + return 0; +} diff --git a/sound/soc/meson/aiu-fifo.c b/sound/soc/meson/aiu-fifo.c new file mode 100644 index 000000000000..d9cede4c33ff --- /dev/null +++ b/sound/soc/meson/aiu-fifo.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include +#include + +#include "aiu-fifo.h" + +#define AIU_MEM_START 0x00 +#define AIU_MEM_RD 0x04 +#define AIU_MEM_END 0x08 +#define AIU_MEM_MASKS 0x0c +#define AIU_MEM_MASK_CH_RD GENMASK(7, 0) +#define AIU_MEM_MASK_CH_MEM GENMASK(15, 8) +#define AIU_MEM_CONTROL 0x10 +#define AIU_MEM_CONTROL_INIT BIT(0) +#define AIU_MEM_CONTROL_FILL_EN BIT(1) +#define AIU_MEM_CONTROL_EMPTY_EN BIT(2) + +static struct snd_soc_dai *aiu_fifo_dai(struct snd_pcm_substream *ss) +{ + struct snd_soc_pcm_runtime *rtd = ss->private_data; + + return asoc_rtd_to_cpu(rtd, 0); +} + +snd_pcm_uframes_t aiu_fifo_pointer(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + struct snd_soc_dai *dai = aiu_fifo_dai(substream); + struct aiu_fifo *fifo = dai->playback_dma_data; + struct snd_pcm_runtime *runtime = substream->runtime; + unsigned int addr; + + snd_soc_component_read(component, fifo->mem_offset + AIU_MEM_RD, + &addr); + + return bytes_to_frames(runtime, addr - (unsigned int)runtime->dma_addr); +} + +static void aiu_fifo_enable(struct snd_soc_dai *dai, bool enable) +{ + struct snd_soc_component *component = dai->component; + struct aiu_fifo *fifo = dai->playback_dma_data; + unsigned int en_mask = (AIU_MEM_CONTROL_FILL_EN | + AIU_MEM_CONTROL_EMPTY_EN); + + snd_soc_component_update_bits(component, + fifo->mem_offset + AIU_MEM_CONTROL, + en_mask, enable ? en_mask : 0); +} + +int aiu_fifo_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai) +{ + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + aiu_fifo_enable(dai, true); + break; + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + case SNDRV_PCM_TRIGGER_STOP: + aiu_fifo_enable(dai, false); + break; + default: + return -EINVAL; + } + + return 0; +} + +int aiu_fifo_prepare(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct aiu_fifo *fifo = dai->playback_dma_data; + + snd_soc_component_update_bits(component, + fifo->mem_offset + AIU_MEM_CONTROL, + AIU_MEM_CONTROL_INIT, + AIU_MEM_CONTROL_INIT); + snd_soc_component_update_bits(component, + fifo->mem_offset + AIU_MEM_CONTROL, + AIU_MEM_CONTROL_INIT, 0); + return 0; +} + +int aiu_fifo_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_pcm_runtime *runtime = substream->runtime; + struct snd_soc_component *component = dai->component; + struct aiu_fifo *fifo = dai->playback_dma_data; + dma_addr_t end; + int ret; + + ret = snd_pcm_lib_malloc_pages(substream, params_buffer_bytes(params)); + if (ret < 0) + return ret; + + /* Setup the fifo boundaries */ + end = runtime->dma_addr + runtime->dma_bytes - fifo->fifo_block; + snd_soc_component_write(component, fifo->mem_offset + AIU_MEM_START, + runtime->dma_addr); + snd_soc_component_write(component, fifo->mem_offset + AIU_MEM_RD, + runtime->dma_addr); + snd_soc_component_write(component, fifo->mem_offset + AIU_MEM_END, + end); + + /* Setup the fifo to read all the memory - no skip */ + snd_soc_component_update_bits(component, + fifo->mem_offset + AIU_MEM_MASKS, + AIU_MEM_MASK_CH_RD | AIU_MEM_MASK_CH_MEM, + FIELD_PREP(AIU_MEM_MASK_CH_RD, 0xff) | + FIELD_PREP(AIU_MEM_MASK_CH_MEM, 0xff)); + + return 0; +} + +int aiu_fifo_hw_free(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + return snd_pcm_lib_free_pages(substream); +} + +static irqreturn_t aiu_fifo_isr(int irq, void *dev_id) +{ + struct snd_pcm_substream *playback = dev_id; + + snd_pcm_period_elapsed(playback); + + return IRQ_HANDLED; +} + +int aiu_fifo_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct aiu_fifo *fifo = dai->playback_dma_data; + int ret; + + snd_soc_set_runtime_hwparams(substream, fifo->pcm); + + /* + * Make sure the buffer and period size are multiple of the fifo burst + * size + */ + ret = snd_pcm_hw_constraint_step(substream->runtime, 0, + SNDRV_PCM_HW_PARAM_BUFFER_BYTES, + fifo->fifo_block); + if (ret) + return ret; + + ret = snd_pcm_hw_constraint_step(substream->runtime, 0, + SNDRV_PCM_HW_PARAM_PERIOD_BYTES, + fifo->fifo_block); + if (ret) + return ret; + + ret = clk_prepare_enable(fifo->pclk); + if (ret) + return ret; + + ret = request_irq(fifo->irq, aiu_fifo_isr, 0, dev_name(dai->dev), + substream); + if (ret) + clk_disable_unprepare(fifo->pclk); + + return ret; +} + +void aiu_fifo_shutdown(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct aiu_fifo *fifo = dai->playback_dma_data; + + free_irq(fifo->irq, substream); + clk_disable_unprepare(fifo->pclk); +} + +int aiu_fifo_pcm_new(struct snd_soc_pcm_runtime *rtd, + struct snd_soc_dai *dai) +{ + struct snd_pcm_substream *substream = + rtd->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + struct snd_card *card = rtd->card->snd_card; + struct aiu_fifo *fifo = dai->playback_dma_data; + size_t size = fifo->pcm->buffer_bytes_max; + + snd_pcm_lib_preallocate_pages(substream, + SNDRV_DMA_TYPE_DEV, + card->dev, size, size); + + return 0; +} + +int aiu_fifo_dai_probe(struct snd_soc_dai *dai) +{ + struct aiu_fifo *fifo; + + fifo = kzalloc(sizeof(*fifo), GFP_KERNEL); + if (!fifo) + return -ENOMEM; + + dai->playback_dma_data = fifo; + + return 0; +} + +int aiu_fifo_dai_remove(struct snd_soc_dai *dai) +{ + kfree(dai->playback_dma_data); + + return 0; +} + diff --git a/sound/soc/meson/aiu-fifo.h b/sound/soc/meson/aiu-fifo.h new file mode 100644 index 000000000000..42ce266677cc --- /dev/null +++ b/sound/soc/meson/aiu-fifo.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/* + * Copyright (c) 2020 BayLibre, SAS. + * Author: Jerome Brunet + */ + +#ifndef _MESON_AIU_FIFO_H +#define _MESON_AIU_FIFO_H + +struct snd_pcm_hardware; +struct snd_soc_component_driver; +struct snd_soc_dai_driver; +struct clk; +struct snd_pcm_ops; +struct snd_pcm_substream; +struct snd_soc_dai; +struct snd_pcm_hw_params; +struct platform_device; + +struct aiu_fifo { + struct snd_pcm_hardware *pcm; + unsigned int mem_offset; + unsigned int fifo_block; + struct clk *pclk; + int irq; +}; + +int aiu_fifo_dai_probe(struct snd_soc_dai *dai); +int aiu_fifo_dai_remove(struct snd_soc_dai *dai); + +snd_pcm_uframes_t aiu_fifo_pointer(struct snd_soc_component *component, + struct snd_pcm_substream *substream); + +int aiu_fifo_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai); +int aiu_fifo_prepare(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai); +int aiu_fifo_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai); +int aiu_fifo_hw_free(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai); +int aiu_fifo_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai); +void aiu_fifo_shutdown(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai); +int aiu_fifo_pcm_new(struct snd_soc_pcm_runtime *rtd, + struct snd_soc_dai *dai); + +#endif /* _MESON_AIU_FIFO_H */ diff --git a/sound/soc/meson/aiu.c b/sound/soc/meson/aiu.c new file mode 100644 index 000000000000..dc35ca79021c --- /dev/null +++ b/sound/soc/meson/aiu.c @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "aiu.h" +#include "aiu-fifo.h" + +#define AIU_I2S_MISC_958_SRC_SHIFT 3 + +static const char * const aiu_spdif_encode_sel_texts[] = { + "SPDIF", "I2S", +}; + +static SOC_ENUM_SINGLE_DECL(aiu_spdif_encode_sel_enum, AIU_I2S_MISC, + AIU_I2S_MISC_958_SRC_SHIFT, + aiu_spdif_encode_sel_texts); + +static const struct snd_kcontrol_new aiu_spdif_encode_mux = + SOC_DAPM_ENUM("SPDIF Buffer Src", aiu_spdif_encode_sel_enum); + +static const struct snd_soc_dapm_widget aiu_cpu_dapm_widgets[] = { + SND_SOC_DAPM_MUX("SPDIF SRC SEL", SND_SOC_NOPM, 0, 0, + &aiu_spdif_encode_mux), +}; + +static const struct snd_soc_dapm_route aiu_cpu_dapm_routes[] = { + { "I2S Encoder Playback", NULL, "I2S FIFO Playback" }, + { "SPDIF SRC SEL", "SPDIF", "SPDIF FIFO Playback" }, + { "SPDIF SRC SEL", "I2S", "I2S FIFO Playback" }, + { "SPDIF Encoder Playback", NULL, "SPDIF SRC SEL" }, +}; + +int aiu_of_xlate_dai_name(struct snd_soc_component *component, + struct of_phandle_args *args, + const char **dai_name, + unsigned int component_id) +{ + struct snd_soc_dai *dai; + int id; + + if (args->args_count != 2) + return -EINVAL; + + if (args->args[0] != component_id) + return -EINVAL; + + id = args->args[1]; + + if (id < 0 || id >= component->num_dai) + return -EINVAL; + + for_each_component_dais(component, dai) { + if (id == 0) + break; + id--; + } + + *dai_name = dai->driver->name; + + return 0; +} + +static int aiu_cpu_of_xlate_dai_name(struct snd_soc_component *component, + struct of_phandle_args *args, + const char **dai_name) +{ + return aiu_of_xlate_dai_name(component, args, dai_name, AIU_CPU); +} + +static int aiu_cpu_component_probe(struct snd_soc_component *component) +{ + struct aiu *aiu = snd_soc_component_get_drvdata(component); + + /* Required for the SPDIF Source control operation */ + return clk_prepare_enable(aiu->i2s.clks[PCLK].clk); +} + +static void aiu_cpu_component_remove(struct snd_soc_component *component) +{ + struct aiu *aiu = snd_soc_component_get_drvdata(component); + + clk_disable_unprepare(aiu->i2s.clks[PCLK].clk); +} + +static const struct snd_soc_component_driver aiu_cpu_component = { + .name = "AIU CPU", + .dapm_widgets = aiu_cpu_dapm_widgets, + .num_dapm_widgets = ARRAY_SIZE(aiu_cpu_dapm_widgets), + .dapm_routes = aiu_cpu_dapm_routes, + .num_dapm_routes = ARRAY_SIZE(aiu_cpu_dapm_routes), + .of_xlate_dai_name = aiu_cpu_of_xlate_dai_name, + .pointer = aiu_fifo_pointer, + .probe = aiu_cpu_component_probe, + .remove = aiu_cpu_component_remove, +}; + +static struct snd_soc_dai_driver aiu_cpu_dai_drv[] = { + [CPU_I2S_FIFO] = { + .name = "I2S FIFO", + .playback = { + .stream_name = "I2S FIFO Playback", + .channels_min = 2, + .channels_max = 8, + .rates = SNDRV_PCM_RATE_CONTINUOUS, + .rate_min = 5512, + .rate_max = 192000, + .formats = AIU_FORMATS, + }, + .ops = &aiu_fifo_i2s_dai_ops, + .pcm_new = aiu_fifo_pcm_new, + .probe = aiu_fifo_i2s_dai_probe, + .remove = aiu_fifo_dai_remove, + }, + [CPU_SPDIF_FIFO] = { + .name = "SPDIF FIFO", + .playback = { + .stream_name = "SPDIF FIFO Playback", + .channels_min = 2, + .channels_max = 2, + .rates = SNDRV_PCM_RATE_CONTINUOUS, + .rate_min = 5512, + .rate_max = 192000, + .formats = AIU_FORMATS, + }, + .ops = &aiu_fifo_spdif_dai_ops, + .pcm_new = aiu_fifo_pcm_new, + .probe = aiu_fifo_spdif_dai_probe, + .remove = aiu_fifo_dai_remove, + }, + [CPU_I2S_ENCODER] = { + .name = "I2S Encoder", + .playback = { + .stream_name = "I2S Encoder Playback", + .channels_min = 2, + .channels_max = 8, + .rates = SNDRV_PCM_RATE_8000_192000, + .formats = AIU_FORMATS, + }, + .ops = &aiu_encoder_i2s_dai_ops, + }, + [CPU_SPDIF_ENCODER] = { + .name = "SPDIF Encoder", + .playback = { + .stream_name = "SPDIF Encoder Playback", + .channels_min = 2, + .channels_max = 2, + .rates = (SNDRV_PCM_RATE_32000 | + SNDRV_PCM_RATE_44100 | + SNDRV_PCM_RATE_48000 | + SNDRV_PCM_RATE_88200 | + SNDRV_PCM_RATE_96000 | + SNDRV_PCM_RATE_176400 | + SNDRV_PCM_RATE_192000), + .formats = AIU_FORMATS, + }, + .ops = &aiu_encoder_spdif_dai_ops, + } +}; + +static const struct regmap_config aiu_regmap_cfg = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = 0x2ac, +}; + +static int aiu_clk_bulk_get(struct device *dev, + const char * const *ids, + unsigned int num, + struct aiu_interface *interface) +{ + struct clk_bulk_data *clks; + int i, ret; + + clks = devm_kcalloc(dev, num, sizeof(*clks), GFP_KERNEL); + if (!clks) + return -ENOMEM; + + for (i = 0; i < num; i++) + clks[i].id = ids[i]; + + ret = devm_clk_bulk_get(dev, num, clks); + if (ret < 0) + return ret; + + interface->clks = clks; + interface->clk_num = num; + return 0; +} + +static const char * const aiu_i2s_ids[] = { + [PCLK] = "i2s_pclk", + [AOCLK] = "i2s_aoclk", + [MCLK] = "i2s_mclk", + [MIXER] = "i2s_mixer", +}; + +static const char * const aiu_spdif_ids[] = { + [PCLK] = "spdif_pclk", + [AOCLK] = "spdif_aoclk", + [MCLK] = "spdif_mclk_sel" +}; + +static int aiu_clk_get(struct device *dev) +{ + struct aiu *aiu = dev_get_drvdata(dev); + int ret; + + aiu->pclk = devm_clk_get(dev, "pclk"); + if (IS_ERR(aiu->pclk)) { + if (PTR_ERR(aiu->pclk) != -EPROBE_DEFER) + dev_err(dev, "Can't get the aiu pclk\n"); + return PTR_ERR(aiu->pclk); + } + + aiu->spdif_mclk = devm_clk_get(dev, "spdif_mclk"); + if (IS_ERR(aiu->spdif_mclk)) { + if (PTR_ERR(aiu->spdif_mclk) != -EPROBE_DEFER) + dev_err(dev, "Can't get the aiu spdif master clock\n"); + return PTR_ERR(aiu->spdif_mclk); + } + + ret = aiu_clk_bulk_get(dev, aiu_i2s_ids, ARRAY_SIZE(aiu_i2s_ids), + &aiu->i2s); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(dev, "Can't get the i2s clocks\n"); + return ret; + } + + ret = aiu_clk_bulk_get(dev, aiu_spdif_ids, ARRAY_SIZE(aiu_spdif_ids), + &aiu->spdif); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(dev, "Can't get the spdif clocks\n"); + return ret; + } + + ret = clk_prepare_enable(aiu->pclk); + if (ret) { + dev_err(dev, "peripheral clock enable failed\n"); + return ret; + } + + ret = devm_add_action_or_reset(dev, + (void(*)(void *))clk_disable_unprepare, + aiu->pclk); + if (ret) + dev_err(dev, "failed to add reset action on pclk"); + + return ret; +} + +static int aiu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + void __iomem *regs; + struct regmap *map; + struct aiu *aiu; + int ret; + + aiu = devm_kzalloc(dev, sizeof(*aiu), GFP_KERNEL); + if (!aiu) + return -ENOMEM; + + aiu->platform = device_get_match_data(dev); + if (!aiu->platform) + return -ENODEV; + + platform_set_drvdata(pdev, aiu); + + ret = device_reset(dev); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to reset device\n"); + return ret; + } + + regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(regs)) + return PTR_ERR(regs); + + map = devm_regmap_init_mmio(dev, regs, &aiu_regmap_cfg); + if (IS_ERR(map)) { + dev_err(dev, "failed to init regmap: %ld\n", + PTR_ERR(map)); + return PTR_ERR(map); + } + + aiu->i2s.irq = platform_get_irq_byname(pdev, "i2s"); + if (aiu->i2s.irq < 0) + return aiu->i2s.irq; + + aiu->spdif.irq = platform_get_irq_byname(pdev, "spdif"); + if (aiu->spdif.irq < 0) + return aiu->spdif.irq; + + ret = aiu_clk_get(dev); + if (ret) + return ret; + + /* Register the cpu component of the aiu */ + ret = snd_soc_register_component(dev, &aiu_cpu_component, + aiu_cpu_dai_drv, + ARRAY_SIZE(aiu_cpu_dai_drv)); + if (ret) { + dev_err(dev, "Failed to register cpu component\n"); + return ret; + } + + /* Register the hdmi codec control component */ + ret = aiu_hdmi_ctrl_register_component(dev); + if (ret) { + dev_err(dev, "Failed to register hdmi control component\n"); + goto err; + } + + /* Register the internal dac control component on gxl */ + if (aiu->platform->has_acodec) { + ret = aiu_acodec_ctrl_register_component(dev); + if (ret) { + dev_err(dev, + "Failed to register acodec control component\n"); + goto err; + } + } + + return 0; +err: + snd_soc_unregister_component(dev); + return ret; +} + +static int aiu_remove(struct platform_device *pdev) +{ + snd_soc_unregister_component(&pdev->dev); + + return 0; +} + +static const struct aiu_platform_data aiu_gxbb_pdata = { + .has_acodec = false, + .has_clk_ctrl_more_i2s_div = true, +}; + +static const struct aiu_platform_data aiu_gxl_pdata = { + .has_acodec = true, + .has_clk_ctrl_more_i2s_div = true, +}; + +static const struct aiu_platform_data aiu_meson8_pdata = { + .has_acodec = false, + .has_clk_ctrl_more_i2s_div = false, +}; + +static const struct of_device_id aiu_of_match[] = { + { .compatible = "amlogic,aiu-gxbb", .data = &aiu_gxbb_pdata }, + { .compatible = "amlogic,aiu-gxl", .data = &aiu_gxl_pdata }, + { .compatible = "amlogic,aiu-meson8", .data = &aiu_meson8_pdata }, + { .compatible = "amlogic,aiu-meson8b", .data = &aiu_meson8_pdata }, + {} +}; +MODULE_DEVICE_TABLE(of, aiu_of_match); + +static struct platform_driver aiu_pdrv = { + .probe = aiu_probe, + .remove = aiu_remove, + .driver = { + .name = "meson-aiu", + .of_match_table = aiu_of_match, + }, +}; +module_platform_driver(aiu_pdrv); + +MODULE_DESCRIPTION("Meson AIU Driver"); +MODULE_AUTHOR("Jerome Brunet "); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/meson/aiu.h b/sound/soc/meson/aiu.h new file mode 100644 index 000000000000..87aa19ac4af3 --- /dev/null +++ b/sound/soc/meson/aiu.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/* + * Copyright (c) 2018 BayLibre, SAS. + * Author: Jerome Brunet + */ + +#ifndef _MESON_AIU_H +#define _MESON_AIU_H + +struct clk; +struct clk_bulk_data; +struct device; +struct of_phandle_args; +struct snd_soc_dai; +struct snd_soc_dai_ops; + +enum aiu_clk_ids { + PCLK = 0, + AOCLK, + MCLK, + MIXER +}; + +struct aiu_interface { + struct clk_bulk_data *clks; + unsigned int clk_num; + int irq; +}; + +struct aiu_platform_data { + bool has_acodec; + bool has_clk_ctrl_more_i2s_div; +}; + +struct aiu { + struct clk *pclk; + struct clk *spdif_mclk; + struct aiu_interface i2s; + struct aiu_interface spdif; + const struct aiu_platform_data *platform; +}; + +#define AIU_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | \ + SNDRV_PCM_FMTBIT_S20_LE | \ + SNDRV_PCM_FMTBIT_S24_LE) + +int aiu_of_xlate_dai_name(struct snd_soc_component *component, + struct of_phandle_args *args, + const char **dai_name, + unsigned int component_id); + +int aiu_hdmi_ctrl_register_component(struct device *dev); +int aiu_acodec_ctrl_register_component(struct device *dev); + +int aiu_fifo_i2s_dai_probe(struct snd_soc_dai *dai); +int aiu_fifo_spdif_dai_probe(struct snd_soc_dai *dai); + +extern const struct snd_soc_dai_ops aiu_fifo_i2s_dai_ops; +extern const struct snd_soc_dai_ops aiu_fifo_spdif_dai_ops; +extern const struct snd_soc_dai_ops aiu_encoder_i2s_dai_ops; +extern const struct snd_soc_dai_ops aiu_encoder_spdif_dai_ops; + +#define AIU_IEC958_BPF 0x000 +#define AIU_958_MISC 0x010 +#define AIU_IEC958_DCU_FF_CTRL 0x01c +#define AIU_958_CHSTAT_L0 0x020 +#define AIU_958_CHSTAT_L1 0x024 +#define AIU_958_CTRL 0x028 +#define AIU_I2S_SOURCE_DESC 0x034 +#define AIU_I2S_DAC_CFG 0x040 +#define AIU_I2S_SYNC 0x044 +#define AIU_I2S_MISC 0x048 +#define AIU_RST_SOFT 0x054 +#define AIU_CLK_CTRL 0x058 +#define AIU_CLK_CTRL_MORE 0x064 +#define AIU_CODEC_DAC_LRCLK_CTRL 0x0a0 +#define AIU_HDMI_CLK_DATA_CTRL 0x0a8 +#define AIU_ACODEC_CTRL 0x0b0 +#define AIU_958_CHSTAT_R0 0x0c0 +#define AIU_958_CHSTAT_R1 0x0c4 +#define AIU_MEM_I2S_START 0x180 +#define AIU_MEM_I2S_MASKS 0x18c +#define AIU_MEM_I2S_CONTROL 0x190 +#define AIU_MEM_IEC958_START 0x194 +#define AIU_MEM_IEC958_CONTROL 0x1a4 +#define AIU_MEM_I2S_BUF_CNTL 0x1d8 +#define AIU_MEM_IEC958_BUF_CNTL 0x1fc + +#endif /* _MESON_AIU_H */ diff --git a/sound/soc/meson/axg-card.c b/sound/soc/meson/axg-card.c index 1f698adde506..af46845f4ef2 100644 --- a/sound/soc/meson/axg-card.c +++ b/sound/soc/meson/axg-card.c @@ -9,11 +9,7 @@ #include #include "axg-tdm.h" - -struct axg_card { - struct snd_soc_card card; - void **link_data; -}; +#include "meson-card.h" struct axg_dai_link_tdm_mask { u32 tx; @@ -41,161 +37,15 @@ static const struct snd_soc_pcm_stream codec_params = { .channels_max = 8, }; -#define PREFIX "amlogic," - -static int axg_card_reallocate_links(struct axg_card *priv, - unsigned int num_links) -{ - struct snd_soc_dai_link *links; - void **ldata; - - links = krealloc(priv->card.dai_link, - num_links * sizeof(*priv->card.dai_link), - GFP_KERNEL | __GFP_ZERO); - ldata = krealloc(priv->link_data, - num_links * sizeof(*priv->link_data), - GFP_KERNEL | __GFP_ZERO); - - if (!links || !ldata) { - dev_err(priv->card.dev, "failed to allocate links\n"); - return -ENOMEM; - } - - priv->card.dai_link = links; - priv->link_data = ldata; - priv->card.num_links = num_links; - return 0; -} - -static int axg_card_parse_dai(struct snd_soc_card *card, - struct device_node *node, - struct device_node **dai_of_node, - const char **dai_name) -{ - struct of_phandle_args args; - int ret; - - if (!dai_name || !dai_of_node || !node) - return -EINVAL; - - ret = of_parse_phandle_with_args(node, "sound-dai", - "#sound-dai-cells", 0, &args); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(card->dev, "can't parse dai %d\n", ret); - return ret; - } - *dai_of_node = args.np; - - return snd_soc_get_dai_name(&args, dai_name); -} - -static int axg_card_set_link_name(struct snd_soc_card *card, - struct snd_soc_dai_link *link, - struct device_node *node, - const char *prefix) -{ - char *name = devm_kasprintf(card->dev, GFP_KERNEL, "%s.%s", - prefix, node->full_name); - if (!name) - return -ENOMEM; - - link->name = name; - link->stream_name = name; - - return 0; -} - -static void axg_card_clean_references(struct axg_card *priv) -{ - struct snd_soc_card *card = &priv->card; - struct snd_soc_dai_link *link; - struct snd_soc_dai_link_component *codec; - struct snd_soc_aux_dev *aux; - int i, j; - - if (card->dai_link) { - for_each_card_prelinks(card, i, link) { - if (link->cpus) - of_node_put(link->cpus->of_node); - for_each_link_codecs(link, j, codec) - of_node_put(codec->of_node); - } - } - - if (card->aux_dev) { - for_each_card_pre_auxs(card, i, aux) - of_node_put(aux->dlc.of_node); - } - - kfree(card->dai_link); - kfree(priv->link_data); -} - -static int axg_card_add_aux_devices(struct snd_soc_card *card) -{ - struct device_node *node = card->dev->of_node; - struct snd_soc_aux_dev *aux; - int num, i; - - num = of_count_phandle_with_args(node, "audio-aux-devs", NULL); - if (num == -ENOENT) { - /* - * It is ok to have no auxiliary devices but for this card it - * is a strange situtation. Let's warn the about it. - */ - dev_warn(card->dev, "card has no auxiliary devices\n"); - return 0; - } else if (num < 0) { - dev_err(card->dev, "error getting auxiliary devices: %d\n", - num); - return num; - } - - aux = devm_kcalloc(card->dev, num, sizeof(*aux), GFP_KERNEL); - if (!aux) - return -ENOMEM; - card->aux_dev = aux; - card->num_aux_devs = num; - - for_each_card_pre_auxs(card, i, aux) { - aux->dlc.of_node = - of_parse_phandle(node, "audio-aux-devs", i); - if (!aux->dlc.of_node) - return -EINVAL; - } - - return 0; -} - static int axg_card_tdm_be_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct axg_card *priv = snd_soc_card_get_drvdata(rtd->card); + struct meson_card *priv = snd_soc_card_get_drvdata(rtd->card); struct axg_dai_link_tdm_data *be = (struct axg_dai_link_tdm_data *)priv->link_data[rtd->num]; - struct snd_soc_dai *codec_dai; - unsigned int mclk; - int ret, i; - if (be->mclk_fs) { - mclk = params_rate(params) * be->mclk_fs; - - for_each_rtd_codec_dai(rtd, i, codec_dai) { - ret = snd_soc_dai_set_sysclk(codec_dai, 0, mclk, - SND_SOC_CLOCK_IN); - if (ret && ret != -ENOTSUPP) - return ret; - } - - ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, 0, mclk, - SND_SOC_CLOCK_OUT); - if (ret && ret != -ENOTSUPP) - return ret; - } - - return 0; + return meson_card_i2s_set_sysclk(substream, params, be->mclk_fs); } static const struct snd_soc_ops axg_card_tdm_be_ops = { @@ -204,13 +54,13 @@ static const struct snd_soc_ops axg_card_tdm_be_ops = { static int axg_card_tdm_dai_init(struct snd_soc_pcm_runtime *rtd) { - struct axg_card *priv = snd_soc_card_get_drvdata(rtd->card); + struct meson_card *priv = snd_soc_card_get_drvdata(rtd->card); struct axg_dai_link_tdm_data *be = (struct axg_dai_link_tdm_data *)priv->link_data[rtd->num]; struct snd_soc_dai *codec_dai; int ret, i; - for_each_rtd_codec_dai(rtd, i, codec_dai) { + for_each_rtd_codec_dais(rtd, i, codec_dai) { ret = snd_soc_dai_set_tdm_slot(codec_dai, be->codec_masks[i].tx, be->codec_masks[i].rx, @@ -222,10 +72,10 @@ static int axg_card_tdm_dai_init(struct snd_soc_pcm_runtime *rtd) } } - ret = axg_tdm_set_tdm_slots(rtd->cpu_dai, be->tx_mask, be->rx_mask, + ret = axg_tdm_set_tdm_slots(asoc_rtd_to_cpu(rtd, 0), be->tx_mask, be->rx_mask, be->slots, be->slot_width); if (ret) { - dev_err(rtd->cpu_dai->dev, "setting tdm link slots failed\n"); + dev_err(asoc_rtd_to_cpu(rtd, 0)->dev, "setting tdm link slots failed\n"); return ret; } @@ -234,16 +84,16 @@ static int axg_card_tdm_dai_init(struct snd_soc_pcm_runtime *rtd) static int axg_card_tdm_dai_lb_init(struct snd_soc_pcm_runtime *rtd) { - struct axg_card *priv = snd_soc_card_get_drvdata(rtd->card); + struct meson_card *priv = snd_soc_card_get_drvdata(rtd->card); struct axg_dai_link_tdm_data *be = (struct axg_dai_link_tdm_data *)priv->link_data[rtd->num]; int ret; /* The loopback rx_mask is the pad tx_mask */ - ret = axg_tdm_set_tdm_slots(rtd->cpu_dai, NULL, be->tx_mask, + ret = axg_tdm_set_tdm_slots(asoc_rtd_to_cpu(rtd, 0), NULL, be->tx_mask, be->slots, be->slot_width); if (ret) { - dev_err(rtd->cpu_dai->dev, "setting tdm link slots failed\n"); + dev_err(asoc_rtd_to_cpu(rtd, 0)->dev, "setting tdm link slots failed\n"); return ret; } @@ -253,14 +103,14 @@ static int axg_card_tdm_dai_lb_init(struct snd_soc_pcm_runtime *rtd) static int axg_card_add_tdm_loopback(struct snd_soc_card *card, int *index) { - struct axg_card *priv = snd_soc_card_get_drvdata(card); + struct meson_card *priv = snd_soc_card_get_drvdata(card); struct snd_soc_dai_link *pad = &card->dai_link[*index]; struct snd_soc_dai_link *lb; struct snd_soc_dai_link_component *dlc; int ret; /* extend links */ - ret = axg_card_reallocate_links(priv, card->num_links + 1); + ret = meson_card_reallocate_links(card, card->num_links + 1); if (ret) return ret; @@ -304,32 +154,6 @@ static int axg_card_add_tdm_loopback(struct snd_soc_card *card, return 0; } -static unsigned int axg_card_parse_daifmt(struct device_node *node, - struct device_node *cpu_node) -{ - struct device_node *bitclkmaster = NULL; - struct device_node *framemaster = NULL; - unsigned int daifmt; - - daifmt = snd_soc_of_parse_daifmt(node, PREFIX, - &bitclkmaster, &framemaster); - daifmt &= ~SND_SOC_DAIFMT_MASTER_MASK; - - /* If no master is provided, default to cpu master */ - if (!bitclkmaster || bitclkmaster == cpu_node) { - daifmt |= (!framemaster || framemaster == cpu_node) ? - SND_SOC_DAIFMT_CBS_CFS : SND_SOC_DAIFMT_CBS_CFM; - } else { - daifmt |= (!framemaster || framemaster == cpu_node) ? - SND_SOC_DAIFMT_CBM_CFS : SND_SOC_DAIFMT_CBM_CFM; - } - - of_node_put(bitclkmaster); - of_node_put(framemaster); - - return daifmt; -} - static int axg_card_parse_cpu_tdm_slots(struct snd_soc_card *card, struct snd_soc_dai_link *link, struct device_node *node, @@ -424,7 +248,7 @@ static int axg_card_parse_tdm(struct snd_soc_card *card, struct device_node *node, int *index) { - struct axg_card *priv = snd_soc_card_get_drvdata(card); + struct meson_card *priv = snd_soc_card_get_drvdata(card); struct snd_soc_dai_link *link = &card->dai_link[*index]; struct axg_dai_link_tdm_data *be; int ret; @@ -438,7 +262,7 @@ static int axg_card_parse_tdm(struct snd_soc_card *card, /* Setup tdm link */ link->ops = &axg_card_tdm_be_ops; link->init = axg_card_tdm_dai_init; - link->dai_fmt = axg_card_parse_daifmt(node, link->cpus->of_node); + link->dai_fmt = meson_card_parse_daifmt(node, link->cpus->of_node); of_property_read_u32(node, "mclk-fs", &be->mclk_fs); @@ -462,97 +286,25 @@ static int axg_card_parse_tdm(struct snd_soc_card *card, return 0; } -static int axg_card_set_be_link(struct snd_soc_card *card, - struct snd_soc_dai_link *link, - struct device_node *node) -{ - struct snd_soc_dai_link_component *codec; - struct device_node *np; - int ret, num_codecs; - - link->no_pcm = 1; - link->dpcm_playback = 1; - link->dpcm_capture = 1; - - num_codecs = of_get_child_count(node); - if (!num_codecs) { - dev_err(card->dev, "be link %s has no codec\n", - node->full_name); - return -EINVAL; - } - - codec = devm_kcalloc(card->dev, num_codecs, sizeof(*codec), GFP_KERNEL); - if (!codec) - return -ENOMEM; - - link->codecs = codec; - link->num_codecs = num_codecs; - - for_each_child_of_node(node, np) { - ret = axg_card_parse_dai(card, np, &codec->of_node, - &codec->dai_name); - if (ret) { - of_node_put(np); - return ret; - } - - codec++; - } - - ret = axg_card_set_link_name(card, link, node, "be"); - if (ret) - dev_err(card->dev, "error setting %pOFn link name\n", np); - - return ret; -} - -static int axg_card_set_fe_link(struct snd_soc_card *card, - struct snd_soc_dai_link *link, - struct device_node *node, - bool is_playback) -{ - struct snd_soc_dai_link_component *codec; - - codec = devm_kzalloc(card->dev, sizeof(*codec), GFP_KERNEL); - if (!codec) - return -ENOMEM; - - link->codecs = codec; - link->num_codecs = 1; - - link->dynamic = 1; - link->dpcm_merged_format = 1; - link->dpcm_merged_chan = 1; - link->dpcm_merged_rate = 1; - link->codecs->dai_name = "snd-soc-dummy-dai"; - link->codecs->name = "snd-soc-dummy"; - - if (is_playback) - link->dpcm_playback = 1; - else - link->dpcm_capture = 1; - - return axg_card_set_link_name(card, link, node, "fe"); -} - static int axg_card_cpu_is_capture_fe(struct device_node *np) { - return of_device_is_compatible(np, PREFIX "axg-toddr"); + return of_device_is_compatible(np, DT_PREFIX "axg-toddr"); } static int axg_card_cpu_is_playback_fe(struct device_node *np) { - return of_device_is_compatible(np, PREFIX "axg-frddr"); + return of_device_is_compatible(np, DT_PREFIX "axg-frddr"); } static int axg_card_cpu_is_tdm_iface(struct device_node *np) { - return of_device_is_compatible(np, PREFIX "axg-tdm-iface"); + return of_device_is_compatible(np, DT_PREFIX "axg-tdm-iface"); } static int axg_card_cpu_is_codec(struct device_node *np) { - return of_device_is_compatible(np, PREFIX "g12a-tohdmitx"); + return of_device_is_compatible(np, DT_PREFIX "g12a-tohdmitx") || + of_device_is_compatible(np, DT_PREFIX "g12a-toacodec"); } static int axg_card_add_link(struct snd_soc_card *card, struct device_node *np, @@ -569,17 +321,17 @@ static int axg_card_add_link(struct snd_soc_card *card, struct device_node *np, dai_link->cpus = cpu; dai_link->num_cpus = 1; - ret = axg_card_parse_dai(card, np, &dai_link->cpus->of_node, - &dai_link->cpus->dai_name); + ret = meson_card_parse_dai(card, np, &dai_link->cpus->of_node, + &dai_link->cpus->dai_name); if (ret) return ret; if (axg_card_cpu_is_playback_fe(dai_link->cpus->of_node)) - ret = axg_card_set_fe_link(card, dai_link, np, true); + ret = meson_card_set_fe_link(card, dai_link, np, true); else if (axg_card_cpu_is_capture_fe(dai_link->cpus->of_node)) - ret = axg_card_set_fe_link(card, dai_link, np, false); + ret = meson_card_set_fe_link(card, dai_link, np, false); else - ret = axg_card_set_be_link(card, dai_link, np); + ret = meson_card_set_be_link(card, dai_link, np); if (ret) return ret; @@ -592,121 +344,21 @@ static int axg_card_add_link(struct snd_soc_card *card, struct device_node *np, return ret; } -static int axg_card_add_links(struct snd_soc_card *card) -{ - struct axg_card *priv = snd_soc_card_get_drvdata(card); - struct device_node *node = card->dev->of_node; - struct device_node *np; - int num, i, ret; - - num = of_get_child_count(node); - if (!num) { - dev_err(card->dev, "card has no links\n"); - return -EINVAL; - } - - ret = axg_card_reallocate_links(priv, num); - if (ret) - return ret; - - i = 0; - for_each_child_of_node(node, np) { - ret = axg_card_add_link(card, np, &i); - if (ret) { - of_node_put(np); - return ret; - } - - i++; - } - - return 0; -} - -static int axg_card_parse_of_optional(struct snd_soc_card *card, - const char *propname, - int (*func)(struct snd_soc_card *c, - const char *p)) -{ - /* If property is not provided, don't fail ... */ - if (!of_property_read_bool(card->dev->of_node, propname)) - return 0; - - /* ... but do fail if it is provided and the parsing fails */ - return func(card, propname); -} +static const struct meson_card_match_data axg_card_match_data = { + .add_link = axg_card_add_link, +}; static const struct of_device_id axg_card_of_match[] = { - { .compatible = "amlogic,axg-sound-card", }, - {} + { + .compatible = "amlogic,axg-sound-card", + .data = &axg_card_match_data, + }, {} }; MODULE_DEVICE_TABLE(of, axg_card_of_match); -static int axg_card_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct axg_card *priv; - int ret; - - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - platform_set_drvdata(pdev, priv); - snd_soc_card_set_drvdata(&priv->card, priv); - - priv->card.owner = THIS_MODULE; - priv->card.dev = dev; - - ret = snd_soc_of_parse_card_name(&priv->card, "model"); - if (ret < 0) - return ret; - - ret = axg_card_parse_of_optional(&priv->card, "audio-routing", - snd_soc_of_parse_audio_routing); - if (ret) { - dev_err(dev, "error while parsing routing\n"); - return ret; - } - - ret = axg_card_parse_of_optional(&priv->card, "audio-widgets", - snd_soc_of_parse_audio_simple_widgets); - if (ret) { - dev_err(dev, "error while parsing widgets\n"); - return ret; - } - - ret = axg_card_add_links(&priv->card); - if (ret) - goto out_err; - - ret = axg_card_add_aux_devices(&priv->card); - if (ret) - goto out_err; - - ret = devm_snd_soc_register_card(dev, &priv->card); - if (ret) - goto out_err; - - return 0; - -out_err: - axg_card_clean_references(priv); - return ret; -} - -static int axg_card_remove(struct platform_device *pdev) -{ - struct axg_card *priv = platform_get_drvdata(pdev); - - axg_card_clean_references(priv); - - return 0; -} - static struct platform_driver axg_card_pdrv = { - .probe = axg_card_probe, - .remove = axg_card_remove, + .probe = meson_card_probe, + .remove = meson_card_remove, .driver = { .name = "axg-sound-card", .of_match_table = axg_card_of_match, diff --git a/sound/soc/meson/axg-fifo.c b/sound/soc/meson/axg-fifo.c index c12b0d5e8ebf..2e9b56b29d31 100644 --- a/sound/soc/meson/axg-fifo.c +++ b/sound/soc/meson/axg-fifo.c @@ -47,7 +47,7 @@ static struct snd_soc_dai *axg_fifo_dai(struct snd_pcm_substream *ss) { struct snd_soc_pcm_runtime *rtd = ss->private_data; - return rtd->cpu_dai; + return asoc_rtd_to_cpu(rtd, 0); } static struct axg_fifo *axg_fifo_data(struct snd_pcm_substream *ss) diff --git a/sound/soc/meson/g12a-toacodec.c b/sound/soc/meson/g12a-toacodec.c new file mode 100644 index 000000000000..9339fabccb79 --- /dev/null +++ b/sound/soc/meson/g12a-toacodec.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "axg-tdm.h" +#include "meson-codec-glue.h" + +#define G12A_TOACODEC_DRV_NAME "g12a-toacodec" + +#define TOACODEC_CTRL0 0x0 +#define CTRL0_ENABLE_SHIFT 31 +#define CTRL0_DAT_SEL_SHIFT 14 +#define CTRL0_DAT_SEL (0x3 << CTRL0_DAT_SEL_SHIFT) +#define CTRL0_LANE_SEL 12 +#define CTRL0_LRCLK_SEL GENMASK(9, 8) +#define CTRL0_BLK_CAP_INV BIT(7) +#define CTRL0_BCLK_O_INV BIT(6) +#define CTRL0_BCLK_SEL GENMASK(5, 4) +#define CTRL0_MCLK_SEL GENMASK(2, 0) + +#define TOACODEC_OUT_CHMAX 2 + +static const char * const g12a_toacodec_mux_texts[] = { + "I2S A", "I2S B", "I2S C", +}; + +static int g12a_toacodec_mux_put_enum(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = + snd_soc_dapm_kcontrol_component(kcontrol); + struct snd_soc_dapm_context *dapm = + snd_soc_dapm_kcontrol_dapm(kcontrol); + struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; + unsigned int mux, changed; + + mux = snd_soc_enum_item_to_val(e, ucontrol->value.enumerated.item[0]); + changed = snd_soc_component_test_bits(component, e->reg, + CTRL0_DAT_SEL, + FIELD_PREP(CTRL0_DAT_SEL, mux)); + + if (!changed) + return 0; + + /* Force disconnect of the mux while updating */ + snd_soc_dapm_mux_update_power(dapm, kcontrol, 0, NULL, NULL); + + snd_soc_component_update_bits(component, e->reg, + CTRL0_DAT_SEL | + CTRL0_LRCLK_SEL | + CTRL0_BCLK_SEL, + FIELD_PREP(CTRL0_DAT_SEL, mux) | + FIELD_PREP(CTRL0_LRCLK_SEL, mux) | + FIELD_PREP(CTRL0_BCLK_SEL, mux)); + + /* + * FIXME: + * On this soc, the glue gets the MCLK directly from the clock + * controller instead of going the through the TDM interface. + * + * Here we assume interface A uses clock A, etc ... While it is + * true for now, it could be different. Instead the glue should + * find out the clock used by the interface and select the same + * source. For that, we will need regmap backed clock mux which + * is a work in progress + */ + snd_soc_component_update_bits(component, e->reg, + CTRL0_MCLK_SEL, + FIELD_PREP(CTRL0_MCLK_SEL, mux)); + + snd_soc_dapm_mux_update_power(dapm, kcontrol, mux, e, NULL); + + return 0; +} + +static SOC_ENUM_SINGLE_DECL(g12a_toacodec_mux_enum, TOACODEC_CTRL0, + CTRL0_DAT_SEL_SHIFT, + g12a_toacodec_mux_texts); + +static const struct snd_kcontrol_new g12a_toacodec_mux = + SOC_DAPM_ENUM_EXT("Source", g12a_toacodec_mux_enum, + snd_soc_dapm_get_enum_double, + g12a_toacodec_mux_put_enum); + +static const struct snd_kcontrol_new g12a_toacodec_out_enable = + SOC_DAPM_SINGLE_AUTODISABLE("Switch", TOACODEC_CTRL0, + CTRL0_ENABLE_SHIFT, 1, 0); + +static const struct snd_soc_dapm_widget g12a_toacodec_widgets[] = { + SND_SOC_DAPM_MUX("SRC", SND_SOC_NOPM, 0, 0, + &g12a_toacodec_mux), + SND_SOC_DAPM_SWITCH("OUT EN", SND_SOC_NOPM, 0, 0, + &g12a_toacodec_out_enable), +}; + +static int g12a_toacodec_input_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct meson_codec_glue_input *data; + int ret; + + ret = meson_codec_glue_input_hw_params(substream, params, dai); + if (ret) + return ret; + + /* The glue will provide 1 lane out of the 4 to the output */ + data = meson_codec_glue_input_get_data(dai); + data->params.channels_min = min_t(unsigned int, TOACODEC_OUT_CHMAX, + data->params.channels_min); + data->params.channels_max = min_t(unsigned int, TOACODEC_OUT_CHMAX, + data->params.channels_max); + + return 0; +} + +static const struct snd_soc_dai_ops g12a_toacodec_input_ops = { + .hw_params = g12a_toacodec_input_hw_params, + .set_fmt = meson_codec_glue_input_set_fmt, +}; + +static const struct snd_soc_dai_ops g12a_toacodec_output_ops = { + .startup = meson_codec_glue_output_startup, +}; + +#define TOACODEC_STREAM(xname, xsuffix, xchmax) \ +{ \ + .stream_name = xname " " xsuffix, \ + .channels_min = 1, \ + .channels_max = (xchmax), \ + .rate_min = 5512, \ + .rate_max = 192000, \ + .formats = AXG_TDM_FORMATS, \ +} + +#define TOACODEC_INPUT(xname, xid) { \ + .name = xname, \ + .id = (xid), \ + .playback = TOACODEC_STREAM(xname, "Playback", 8), \ + .ops = &g12a_toacodec_input_ops, \ + .probe = meson_codec_glue_input_dai_probe, \ + .remove = meson_codec_glue_input_dai_remove, \ +} + +#define TOACODEC_OUTPUT(xname, xid) { \ + .name = xname, \ + .id = (xid), \ + .capture = TOACODEC_STREAM(xname, "Capture", TOACODEC_OUT_CHMAX), \ + .ops = &g12a_toacodec_output_ops, \ +} + +static struct snd_soc_dai_driver g12a_toacodec_dai_drv[] = { + TOACODEC_INPUT("IN A", TOACODEC_IN_A), + TOACODEC_INPUT("IN B", TOACODEC_IN_B), + TOACODEC_INPUT("IN C", TOACODEC_IN_C), + TOACODEC_OUTPUT("OUT", TOACODEC_OUT), +}; + +static int g12a_toacodec_component_probe(struct snd_soc_component *c) +{ + /* Initialize the static clock parameters */ + return snd_soc_component_write(c, TOACODEC_CTRL0, + CTRL0_BLK_CAP_INV); +} + +static const struct snd_soc_dapm_route g12a_toacodec_routes[] = { + { "SRC", "I2S A", "IN A Playback" }, + { "SRC", "I2S B", "IN B Playback" }, + { "SRC", "I2S C", "IN C Playback" }, + { "OUT EN", "Switch", "SRC" }, + { "OUT Capture", NULL, "OUT EN" }, +}; + +static const struct snd_kcontrol_new g12a_toacodec_controls[] = { + SOC_SINGLE("Lane Select", TOACODEC_CTRL0, CTRL0_LANE_SEL, 3, 0), +}; + +static const struct snd_soc_component_driver g12a_toacodec_component_drv = { + .probe = g12a_toacodec_component_probe, + .controls = g12a_toacodec_controls, + .num_controls = ARRAY_SIZE(g12a_toacodec_controls), + .dapm_widgets = g12a_toacodec_widgets, + .num_dapm_widgets = ARRAY_SIZE(g12a_toacodec_widgets), + .dapm_routes = g12a_toacodec_routes, + .num_dapm_routes = ARRAY_SIZE(g12a_toacodec_routes), + .endianness = 1, + .non_legacy_dai_naming = 1, +}; + +static const struct regmap_config g12a_toacodec_regmap_cfg = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +static const struct of_device_id g12a_toacodec_of_match[] = { + { .compatible = "amlogic,g12a-toacodec", }, + {} +}; +MODULE_DEVICE_TABLE(of, g12a_toacodec_of_match); + +static int g12a_toacodec_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + void __iomem *regs; + struct regmap *map; + int ret; + + ret = device_reset(dev); + if (ret) + return ret; + + regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(regs)) + return PTR_ERR(regs); + + map = devm_regmap_init_mmio(dev, regs, &g12a_toacodec_regmap_cfg); + if (IS_ERR(map)) { + dev_err(dev, "failed to init regmap: %ld\n", + PTR_ERR(map)); + return PTR_ERR(map); + } + + return devm_snd_soc_register_component(dev, + &g12a_toacodec_component_drv, g12a_toacodec_dai_drv, + ARRAY_SIZE(g12a_toacodec_dai_drv)); +} + +static struct platform_driver g12a_toacodec_pdrv = { + .driver = { + .name = G12A_TOACODEC_DRV_NAME, + .of_match_table = g12a_toacodec_of_match, + }, + .probe = g12a_toacodec_probe, +}; +module_platform_driver(g12a_toacodec_pdrv); + +MODULE_AUTHOR("Jerome Brunet "); +MODULE_DESCRIPTION("Amlogic G12a To Internal DAC Codec Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/meson/g12a-tohdmitx.c b/sound/soc/meson/g12a-tohdmitx.c index 8a0db28a6a40..9b2b59536ced 100644 --- a/sound/soc/meson/g12a-tohdmitx.c +++ b/sound/soc/meson/g12a-tohdmitx.c @@ -13,112 +13,51 @@ #include #include +#include "meson-codec-glue.h" #define G12A_TOHDMITX_DRV_NAME "g12a-tohdmitx" #define TOHDMITX_CTRL0 0x0 #define CTRL0_ENABLE_SHIFT 31 -#define CTRL0_I2S_DAT_SEL GENMASK(13, 12) +#define CTRL0_I2S_DAT_SEL_SHIFT 12 +#define CTRL0_I2S_DAT_SEL (0x3 << CTRL0_I2S_DAT_SEL_SHIFT) #define CTRL0_I2S_LRCLK_SEL GENMASK(9, 8) #define CTRL0_I2S_BLK_CAP_INV BIT(7) #define CTRL0_I2S_BCLK_O_INV BIT(6) #define CTRL0_I2S_BCLK_SEL GENMASK(5, 4) #define CTRL0_SPDIF_CLK_CAP_INV BIT(3) #define CTRL0_SPDIF_CLK_O_INV BIT(2) -#define CTRL0_SPDIF_SEL BIT(1) +#define CTRL0_SPDIF_SEL_SHIFT 1 +#define CTRL0_SPDIF_SEL (0x1 << CTRL0_SPDIF_SEL_SHIFT) #define CTRL0_SPDIF_CLK_SEL BIT(0) -struct g12a_tohdmitx_input { - struct snd_soc_pcm_stream params; - unsigned int fmt; -}; - -static struct snd_soc_dapm_widget * -g12a_tohdmitx_get_input(struct snd_soc_dapm_widget *w) -{ - struct snd_soc_dapm_path *p = NULL; - struct snd_soc_dapm_widget *in; - - snd_soc_dapm_widget_for_each_source_path(w, p) { - if (!p->connect) - continue; - - /* Check that we still are in the same component */ - if (snd_soc_dapm_to_component(w->dapm) != - snd_soc_dapm_to_component(p->source->dapm)) - continue; - - if (p->source->id == snd_soc_dapm_dai_in) - return p->source; - - in = g12a_tohdmitx_get_input(p->source); - if (in) - return in; - } - - return NULL; -} - -static struct g12a_tohdmitx_input * -g12a_tohdmitx_get_input_data(struct snd_soc_dapm_widget *w) -{ - struct snd_soc_dapm_widget *in = - g12a_tohdmitx_get_input(w); - struct snd_soc_dai *dai; - - if (WARN_ON(!in)) - return NULL; - - dai = in->priv; - - return dai->playback_dma_data; -} - static const char * const g12a_tohdmitx_i2s_mux_texts[] = { "I2S A", "I2S B", "I2S C", }; -static SOC_ENUM_SINGLE_EXT_DECL(g12a_tohdmitx_i2s_mux_enum, - g12a_tohdmitx_i2s_mux_texts); - -static int g12a_tohdmitx_get_input_val(struct snd_soc_component *component, - unsigned int mask) -{ - unsigned int val; - - snd_soc_component_read(component, TOHDMITX_CTRL0, &val); - return (val & mask) >> __ffs(mask); -} - -static int g12a_tohdmitx_i2s_mux_get_enum(struct snd_kcontrol *kcontrol, - struct snd_ctl_elem_value *ucontrol) -{ - struct snd_soc_component *component = - snd_soc_dapm_kcontrol_component(kcontrol); - - ucontrol->value.enumerated.item[0] = - g12a_tohdmitx_get_input_val(component, CTRL0_I2S_DAT_SEL); - - return 0; -} - static int g12a_tohdmitx_i2s_mux_put_enum(struct snd_kcontrol *kcontrol, - struct snd_ctl_elem_value *ucontrol) + struct snd_ctl_elem_value *ucontrol) { struct snd_soc_component *component = snd_soc_dapm_kcontrol_component(kcontrol); struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kcontrol); struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; - unsigned int mux = ucontrol->value.enumerated.item[0]; - unsigned int val = g12a_tohdmitx_get_input_val(component, - CTRL0_I2S_DAT_SEL); + unsigned int mux, changed; + + mux = snd_soc_enum_item_to_val(e, ucontrol->value.enumerated.item[0]); + changed = snd_soc_component_test_bits(component, e->reg, + CTRL0_I2S_DAT_SEL, + FIELD_PREP(CTRL0_I2S_DAT_SEL, + mux)); + + if (!changed) + return 0; /* Force disconnect of the mux while updating */ - if (val != mux) - snd_soc_dapm_mux_update_power(dapm, kcontrol, 0, NULL, NULL); + snd_soc_dapm_mux_update_power(dapm, kcontrol, 0, NULL, NULL); - snd_soc_component_update_bits(component, TOHDMITX_CTRL0, + snd_soc_component_update_bits(component, e->reg, CTRL0_I2S_DAT_SEL | CTRL0_I2S_LRCLK_SEL | CTRL0_I2S_BCLK_SEL, @@ -131,30 +70,19 @@ static int g12a_tohdmitx_i2s_mux_put_enum(struct snd_kcontrol *kcontrol, return 0; } +static SOC_ENUM_SINGLE_DECL(g12a_tohdmitx_i2s_mux_enum, TOHDMITX_CTRL0, + CTRL0_I2S_DAT_SEL_SHIFT, + g12a_tohdmitx_i2s_mux_texts); + static const struct snd_kcontrol_new g12a_tohdmitx_i2s_mux = SOC_DAPM_ENUM_EXT("I2S Source", g12a_tohdmitx_i2s_mux_enum, - g12a_tohdmitx_i2s_mux_get_enum, + snd_soc_dapm_get_enum_double, g12a_tohdmitx_i2s_mux_put_enum); static const char * const g12a_tohdmitx_spdif_mux_texts[] = { "SPDIF A", "SPDIF B", }; -static SOC_ENUM_SINGLE_EXT_DECL(g12a_tohdmitx_spdif_mux_enum, - g12a_tohdmitx_spdif_mux_texts); - -static int g12a_tohdmitx_spdif_mux_get_enum(struct snd_kcontrol *kcontrol, - struct snd_ctl_elem_value *ucontrol) -{ - struct snd_soc_component *component = - snd_soc_dapm_kcontrol_component(kcontrol); - - ucontrol->value.enumerated.item[0] = - g12a_tohdmitx_get_input_val(component, CTRL0_SPDIF_SEL); - - return 0; -} - static int g12a_tohdmitx_spdif_mux_put_enum(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { @@ -163,13 +91,18 @@ static int g12a_tohdmitx_spdif_mux_put_enum(struct snd_kcontrol *kcontrol, struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kcontrol); struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; - unsigned int mux = ucontrol->value.enumerated.item[0]; - unsigned int val = g12a_tohdmitx_get_input_val(component, - CTRL0_SPDIF_SEL); + unsigned int mux, changed; + + mux = snd_soc_enum_item_to_val(e, ucontrol->value.enumerated.item[0]); + changed = snd_soc_component_test_bits(component, TOHDMITX_CTRL0, + CTRL0_SPDIF_SEL, + FIELD_PREP(CTRL0_SPDIF_SEL, mux)); + + if (!changed) + return 0; /* Force disconnect of the mux while updating */ - if (val != mux) - snd_soc_dapm_mux_update_power(dapm, kcontrol, 0, NULL, NULL); + snd_soc_dapm_mux_update_power(dapm, kcontrol, 0, NULL, NULL); snd_soc_component_update_bits(component, TOHDMITX_CTRL0, CTRL0_SPDIF_SEL | @@ -182,9 +115,13 @@ static int g12a_tohdmitx_spdif_mux_put_enum(struct snd_kcontrol *kcontrol, return 0; } +static SOC_ENUM_SINGLE_DECL(g12a_tohdmitx_spdif_mux_enum, TOHDMITX_CTRL0, + CTRL0_SPDIF_SEL_SHIFT, + g12a_tohdmitx_spdif_mux_texts); + static const struct snd_kcontrol_new g12a_tohdmitx_spdif_mux = SOC_DAPM_ENUM_EXT("SPDIF Source", g12a_tohdmitx_spdif_mux_enum, - g12a_tohdmitx_spdif_mux_get_enum, + snd_soc_dapm_get_enum_double, g12a_tohdmitx_spdif_mux_put_enum); static const struct snd_kcontrol_new g12a_tohdmitx_out_enable = @@ -202,83 +139,13 @@ static const struct snd_soc_dapm_widget g12a_tohdmitx_widgets[] = { &g12a_tohdmitx_out_enable), }; -static int g12a_tohdmitx_input_probe(struct snd_soc_dai *dai) -{ - struct g12a_tohdmitx_input *data; - - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; - - dai->playback_dma_data = data; - return 0; -} - -static int g12a_tohdmitx_input_remove(struct snd_soc_dai *dai) -{ - kfree(dai->playback_dma_data); - return 0; -} - -static int g12a_tohdmitx_input_hw_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *params, - struct snd_soc_dai *dai) -{ - struct g12a_tohdmitx_input *data = dai->playback_dma_data; - - data->params.rates = snd_pcm_rate_to_rate_bit(params_rate(params)); - data->params.rate_min = params_rate(params); - data->params.rate_max = params_rate(params); - data->params.formats = 1 << params_format(params); - data->params.channels_min = params_channels(params); - data->params.channels_max = params_channels(params); - data->params.sig_bits = dai->driver->playback.sig_bits; - - return 0; -} - - -static int g12a_tohdmitx_input_set_fmt(struct snd_soc_dai *dai, - unsigned int fmt) -{ - struct g12a_tohdmitx_input *data = dai->playback_dma_data; - - /* Save the source stream format for the downstream link */ - data->fmt = fmt; - return 0; -} - -static int g12a_tohdmitx_output_startup(struct snd_pcm_substream *substream, - struct snd_soc_dai *dai) -{ - struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct g12a_tohdmitx_input *in_data = - g12a_tohdmitx_get_input_data(dai->capture_widget); - - if (!in_data) - return -ENODEV; - - if (WARN_ON(!rtd->dai_link->params)) { - dev_warn(dai->dev, "codec2codec link expected\n"); - return -EINVAL; - } - - /* Replace link params with the input params */ - rtd->dai_link->params = &in_data->params; - - if (!in_data->fmt) - return 0; - - return snd_soc_runtime_set_dai_fmt(rtd, in_data->fmt); -} - static const struct snd_soc_dai_ops g12a_tohdmitx_input_ops = { - .hw_params = g12a_tohdmitx_input_hw_params, - .set_fmt = g12a_tohdmitx_input_set_fmt, + .hw_params = meson_codec_glue_input_hw_params, + .set_fmt = meson_codec_glue_input_set_fmt, }; static const struct snd_soc_dai_ops g12a_tohdmitx_output_ops = { - .startup = g12a_tohdmitx_output_startup, + .startup = meson_codec_glue_output_startup, }; #define TOHDMITX_SPDIF_FORMATS \ @@ -305,8 +172,8 @@ static const struct snd_soc_dai_ops g12a_tohdmitx_output_ops = { .id = (xid), \ .playback = TOHDMITX_STREAM(xname, "Playback", xfmt, xchmax), \ .ops = &g12a_tohdmitx_input_ops, \ - .probe = g12a_tohdmitx_input_probe, \ - .remove = g12a_tohdmitx_input_remove, \ + .probe = meson_codec_glue_input_dai_probe, \ + .remove = meson_codec_glue_input_dai_remove, \ } #define TOHDMITX_OUT(xname, xid, xfmt, xchmax) { \ diff --git a/sound/soc/meson/gx-card.c b/sound/soc/meson/gx-card.c new file mode 100644 index 000000000000..7b01dcb73e5e --- /dev/null +++ b/sound/soc/meson/gx-card.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include + +#include "meson-card.h" + +struct gx_dai_link_i2s_data { + unsigned int mclk_fs; +}; + +/* + * Base params for the codec to codec links + * Those will be over-written by the CPU side of the link + */ +static const struct snd_soc_pcm_stream codec_params = { + .formats = SNDRV_PCM_FMTBIT_S24_LE, + .rate_min = 5525, + .rate_max = 192000, + .channels_min = 1, + .channels_max = 8, +}; + +static int gx_card_i2s_be_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct meson_card *priv = snd_soc_card_get_drvdata(rtd->card); + struct gx_dai_link_i2s_data *be = + (struct gx_dai_link_i2s_data *)priv->link_data[rtd->num]; + + return meson_card_i2s_set_sysclk(substream, params, be->mclk_fs); +} + +static const struct snd_soc_ops gx_card_i2s_be_ops = { + .hw_params = gx_card_i2s_be_hw_params, +}; + +static int gx_card_parse_i2s(struct snd_soc_card *card, + struct device_node *node, + int *index) +{ + struct meson_card *priv = snd_soc_card_get_drvdata(card); + struct snd_soc_dai_link *link = &card->dai_link[*index]; + struct gx_dai_link_i2s_data *be; + + /* Allocate i2s link parameters */ + be = devm_kzalloc(card->dev, sizeof(*be), GFP_KERNEL); + if (!be) + return -ENOMEM; + priv->link_data[*index] = be; + + /* Setup i2s link */ + link->ops = &gx_card_i2s_be_ops; + link->dai_fmt = meson_card_parse_daifmt(node, link->cpus->of_node); + + of_property_read_u32(node, "mclk-fs", &be->mclk_fs); + + return 0; +} + +static int gx_card_cpu_identify(struct snd_soc_dai_link_component *c, + char *match) +{ + if (of_device_is_compatible(c->of_node, DT_PREFIX "aiu")) { + if (strstr(c->dai_name, match)) + return 1; + } + + /* dai not matched */ + return 0; +} + +static int gx_card_add_link(struct snd_soc_card *card, struct device_node *np, + int *index) +{ + struct snd_soc_dai_link *dai_link = &card->dai_link[*index]; + struct snd_soc_dai_link_component *cpu; + int ret; + + cpu = devm_kzalloc(card->dev, sizeof(*cpu), GFP_KERNEL); + if (!cpu) + return -ENOMEM; + + dai_link->cpus = cpu; + dai_link->num_cpus = 1; + + ret = meson_card_parse_dai(card, np, &dai_link->cpus->of_node, + &dai_link->cpus->dai_name); + if (ret) + return ret; + + if (gx_card_cpu_identify(dai_link->cpus, "FIFO")) + ret = meson_card_set_fe_link(card, dai_link, np, true); + else + ret = meson_card_set_be_link(card, dai_link, np); + + if (ret) + return ret; + + /* Check if the cpu is the i2s encoder and parse i2s data */ + if (gx_card_cpu_identify(dai_link->cpus, "I2S Encoder")) + ret = gx_card_parse_i2s(card, np, index); + + /* Or apply codec to codec params if necessary */ + else if (gx_card_cpu_identify(dai_link->cpus, "CODEC CTRL")) + dai_link->params = &codec_params; + + return ret; +} + +static const struct meson_card_match_data gx_card_match_data = { + .add_link = gx_card_add_link, +}; + +static const struct of_device_id gx_card_of_match[] = { + { + .compatible = "amlogic,gx-sound-card", + .data = &gx_card_match_data, + }, {} +}; +MODULE_DEVICE_TABLE(of, gx_card_of_match); + +static struct platform_driver gx_card_pdrv = { + .probe = meson_card_probe, + .remove = meson_card_remove, + .driver = { + .name = "gx-sound-card", + .of_match_table = gx_card_of_match, + }, +}; +module_platform_driver(gx_card_pdrv); + +MODULE_DESCRIPTION("Amlogic GX ALSA machine driver"); +MODULE_AUTHOR("Jerome Brunet "); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/meson/meson-card-utils.c b/sound/soc/meson/meson-card-utils.c new file mode 100644 index 000000000000..2ca8c98e204f --- /dev/null +++ b/sound/soc/meson/meson-card-utils.c @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include + +#include "meson-card.h" + +int meson_card_i2s_set_sysclk(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + unsigned int mclk_fs) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *codec_dai; + unsigned int mclk; + int ret, i; + + if (!mclk_fs) + return 0; + + mclk = params_rate(params) * mclk_fs; + + for_each_rtd_codec_dais(rtd, i, codec_dai) { + ret = snd_soc_dai_set_sysclk(codec_dai, 0, mclk, + SND_SOC_CLOCK_IN); + if (ret && ret != -ENOTSUPP) + return ret; + } + + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(rtd, 0), 0, mclk, + SND_SOC_CLOCK_OUT); + if (ret && ret != -ENOTSUPP) + return ret; + + return 0; +} +EXPORT_SYMBOL_GPL(meson_card_i2s_set_sysclk); + +int meson_card_reallocate_links(struct snd_soc_card *card, + unsigned int num_links) +{ + struct meson_card *priv = snd_soc_card_get_drvdata(card); + struct snd_soc_dai_link *links; + void **ldata; + + links = krealloc(priv->card.dai_link, + num_links * sizeof(*priv->card.dai_link), + GFP_KERNEL | __GFP_ZERO); + ldata = krealloc(priv->link_data, + num_links * sizeof(*priv->link_data), + GFP_KERNEL | __GFP_ZERO); + + if (!links || !ldata) { + dev_err(priv->card.dev, "failed to allocate links\n"); + return -ENOMEM; + } + + priv->card.dai_link = links; + priv->link_data = ldata; + priv->card.num_links = num_links; + return 0; +} +EXPORT_SYMBOL_GPL(meson_card_reallocate_links); + +int meson_card_parse_dai(struct snd_soc_card *card, + struct device_node *node, + struct device_node **dai_of_node, + const char **dai_name) +{ + struct of_phandle_args args; + int ret; + + if (!dai_name || !dai_of_node || !node) + return -EINVAL; + + ret = of_parse_phandle_with_args(node, "sound-dai", + "#sound-dai-cells", 0, &args); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(card->dev, "can't parse dai %d\n", ret); + return ret; + } + *dai_of_node = args.np; + + return snd_soc_get_dai_name(&args, dai_name); +} +EXPORT_SYMBOL_GPL(meson_card_parse_dai); + +static int meson_card_set_link_name(struct snd_soc_card *card, + struct snd_soc_dai_link *link, + struct device_node *node, + const char *prefix) +{ + char *name = devm_kasprintf(card->dev, GFP_KERNEL, "%s.%s", + prefix, node->full_name); + if (!name) + return -ENOMEM; + + link->name = name; + link->stream_name = name; + + return 0; +} + +unsigned int meson_card_parse_daifmt(struct device_node *node, + struct device_node *cpu_node) +{ + struct device_node *bitclkmaster = NULL; + struct device_node *framemaster = NULL; + unsigned int daifmt; + + daifmt = snd_soc_of_parse_daifmt(node, DT_PREFIX, + &bitclkmaster, &framemaster); + daifmt &= ~SND_SOC_DAIFMT_MASTER_MASK; + + /* If no master is provided, default to cpu master */ + if (!bitclkmaster || bitclkmaster == cpu_node) { + daifmt |= (!framemaster || framemaster == cpu_node) ? + SND_SOC_DAIFMT_CBS_CFS : SND_SOC_DAIFMT_CBS_CFM; + } else { + daifmt |= (!framemaster || framemaster == cpu_node) ? + SND_SOC_DAIFMT_CBM_CFS : SND_SOC_DAIFMT_CBM_CFM; + } + + of_node_put(bitclkmaster); + of_node_put(framemaster); + + return daifmt; +} +EXPORT_SYMBOL_GPL(meson_card_parse_daifmt); + +int meson_card_set_be_link(struct snd_soc_card *card, + struct snd_soc_dai_link *link, + struct device_node *node) +{ + struct snd_soc_dai_link_component *codec; + struct device_node *np; + int ret, num_codecs; + + link->no_pcm = 1; + link->dpcm_playback = 1; + link->dpcm_capture = 1; + + num_codecs = of_get_child_count(node); + if (!num_codecs) { + dev_err(card->dev, "be link %s has no codec\n", + node->full_name); + return -EINVAL; + } + + codec = devm_kcalloc(card->dev, num_codecs, sizeof(*codec), GFP_KERNEL); + if (!codec) + return -ENOMEM; + + link->codecs = codec; + link->num_codecs = num_codecs; + + for_each_child_of_node(node, np) { + ret = meson_card_parse_dai(card, np, &codec->of_node, + &codec->dai_name); + if (ret) { + of_node_put(np); + return ret; + } + + codec++; + } + + ret = meson_card_set_link_name(card, link, node, "be"); + if (ret) + dev_err(card->dev, "error setting %pOFn link name\n", np); + + return ret; +} +EXPORT_SYMBOL_GPL(meson_card_set_be_link); + +int meson_card_set_fe_link(struct snd_soc_card *card, + struct snd_soc_dai_link *link, + struct device_node *node, + bool is_playback) +{ + struct snd_soc_dai_link_component *codec; + + codec = devm_kzalloc(card->dev, sizeof(*codec), GFP_KERNEL); + if (!codec) + return -ENOMEM; + + link->codecs = codec; + link->num_codecs = 1; + + link->dynamic = 1; + link->dpcm_merged_format = 1; + link->dpcm_merged_chan = 1; + link->dpcm_merged_rate = 1; + link->codecs->dai_name = "snd-soc-dummy-dai"; + link->codecs->name = "snd-soc-dummy"; + + if (is_playback) + link->dpcm_playback = 1; + else + link->dpcm_capture = 1; + + return meson_card_set_link_name(card, link, node, "fe"); +} +EXPORT_SYMBOL_GPL(meson_card_set_fe_link); + +static int meson_card_add_links(struct snd_soc_card *card) +{ + struct meson_card *priv = snd_soc_card_get_drvdata(card); + struct device_node *node = card->dev->of_node; + struct device_node *np; + int num, i, ret; + + num = of_get_child_count(node); + if (!num) { + dev_err(card->dev, "card has no links\n"); + return -EINVAL; + } + + ret = meson_card_reallocate_links(card, num); + if (ret) + return ret; + + i = 0; + for_each_child_of_node(node, np) { + ret = priv->match_data->add_link(card, np, &i); + if (ret) { + of_node_put(np); + return ret; + } + + i++; + } + + return 0; +} + +static int meson_card_parse_of_optional(struct snd_soc_card *card, + const char *propname, + int (*func)(struct snd_soc_card *c, + const char *p)) +{ + /* If property is not provided, don't fail ... */ + if (!of_property_read_bool(card->dev->of_node, propname)) + return 0; + + /* ... but do fail if it is provided and the parsing fails */ + return func(card, propname); +} + +static int meson_card_add_aux_devices(struct snd_soc_card *card) +{ + struct device_node *node = card->dev->of_node; + struct snd_soc_aux_dev *aux; + int num, i; + + num = of_count_phandle_with_args(node, "audio-aux-devs", NULL); + if (num == -ENOENT) { + return 0; + } else if (num < 0) { + dev_err(card->dev, "error getting auxiliary devices: %d\n", + num); + return num; + } + + aux = devm_kcalloc(card->dev, num, sizeof(*aux), GFP_KERNEL); + if (!aux) + return -ENOMEM; + card->aux_dev = aux; + card->num_aux_devs = num; + + for_each_card_pre_auxs(card, i, aux) { + aux->dlc.of_node = + of_parse_phandle(node, "audio-aux-devs", i); + if (!aux->dlc.of_node) + return -EINVAL; + } + + return 0; +} + +static void meson_card_clean_references(struct meson_card *priv) +{ + struct snd_soc_card *card = &priv->card; + struct snd_soc_dai_link *link; + struct snd_soc_dai_link_component *codec; + struct snd_soc_aux_dev *aux; + int i, j; + + if (card->dai_link) { + for_each_card_prelinks(card, i, link) { + if (link->cpus) + of_node_put(link->cpus->of_node); + for_each_link_codecs(link, j, codec) + of_node_put(codec->of_node); + } + } + + if (card->aux_dev) { + for_each_card_pre_auxs(card, i, aux) + of_node_put(aux->dlc.of_node); + } + + kfree(card->dai_link); + kfree(priv->link_data); +} + +int meson_card_probe(struct platform_device *pdev) +{ + const struct meson_card_match_data *data; + struct device *dev = &pdev->dev; + struct meson_card *priv; + int ret; + + data = of_device_get_match_data(dev); + if (!data) { + dev_err(dev, "failed to match device\n"); + return -ENODEV; + } + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + platform_set_drvdata(pdev, priv); + snd_soc_card_set_drvdata(&priv->card, priv); + + priv->card.owner = THIS_MODULE; + priv->card.dev = dev; + priv->match_data = data; + + ret = snd_soc_of_parse_card_name(&priv->card, "model"); + if (ret < 0) + return ret; + + ret = meson_card_parse_of_optional(&priv->card, "audio-routing", + snd_soc_of_parse_audio_routing); + if (ret) { + dev_err(dev, "error while parsing routing\n"); + return ret; + } + + ret = meson_card_parse_of_optional(&priv->card, "audio-widgets", + snd_soc_of_parse_audio_simple_widgets); + if (ret) { + dev_err(dev, "error while parsing widgets\n"); + return ret; + } + + ret = meson_card_add_links(&priv->card); + if (ret) + goto out_err; + + ret = meson_card_add_aux_devices(&priv->card); + if (ret) + goto out_err; + + ret = devm_snd_soc_register_card(dev, &priv->card); + if (ret) + goto out_err; + + return 0; + +out_err: + meson_card_clean_references(priv); + return ret; +} +EXPORT_SYMBOL_GPL(meson_card_probe); + +int meson_card_remove(struct platform_device *pdev) +{ + struct meson_card *priv = platform_get_drvdata(pdev); + + meson_card_clean_references(priv); + + return 0; +} +EXPORT_SYMBOL_GPL(meson_card_remove); + +MODULE_DESCRIPTION("Amlogic Sound Card Utils"); +MODULE_AUTHOR("Jerome Brunet "); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/meson/meson-card.h b/sound/soc/meson/meson-card.h new file mode 100644 index 000000000000..74314071c80d --- /dev/null +++ b/sound/soc/meson/meson-card.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2020 BayLibre, SAS. + * Author: Jerome Brunet + */ + +#ifndef _MESON_SND_CARD_H +#define _MESON_SND_CARD_H + +struct device_node; +struct platform_device; + +struct snd_soc_card; +struct snd_pcm_substream; +struct snd_pcm_hw_params; + +#define DT_PREFIX "amlogic," + +struct meson_card_match_data { + int (*add_link)(struct snd_soc_card *card, + struct device_node *node, + int *index); +}; + +struct meson_card { + const struct meson_card_match_data *match_data; + struct snd_soc_card card; + void **link_data; +}; + +unsigned int meson_card_parse_daifmt(struct device_node *node, + struct device_node *cpu_node); + +int meson_card_i2s_set_sysclk(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + unsigned int mclk_fs); + +int meson_card_reallocate_links(struct snd_soc_card *card, + unsigned int num_links); +int meson_card_parse_dai(struct snd_soc_card *card, + struct device_node *node, + struct device_node **dai_of_node, + const char **dai_name); +int meson_card_set_be_link(struct snd_soc_card *card, + struct snd_soc_dai_link *link, + struct device_node *node); +int meson_card_set_fe_link(struct snd_soc_card *card, + struct snd_soc_dai_link *link, + struct device_node *node, + bool is_playback); + +int meson_card_probe(struct platform_device *pdev); +int meson_card_remove(struct platform_device *pdev); + +#endif /* _MESON_SND_CARD_H */ diff --git a/sound/soc/meson/meson-codec-glue.c b/sound/soc/meson/meson-codec-glue.c new file mode 100644 index 000000000000..524a33472337 --- /dev/null +++ b/sound/soc/meson/meson-codec-glue.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2019 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include + +#include "meson-codec-glue.h" + +static struct snd_soc_dapm_widget * +meson_codec_glue_get_input(struct snd_soc_dapm_widget *w) +{ + struct snd_soc_dapm_path *p = NULL; + struct snd_soc_dapm_widget *in; + + snd_soc_dapm_widget_for_each_source_path(w, p) { + if (!p->connect) + continue; + + /* Check that we still are in the same component */ + if (snd_soc_dapm_to_component(w->dapm) != + snd_soc_dapm_to_component(p->source->dapm)) + continue; + + if (p->source->id == snd_soc_dapm_dai_in) + return p->source; + + in = meson_codec_glue_get_input(p->source); + if (in) + return in; + } + + return NULL; +} + +static void meson_codec_glue_input_set_data(struct snd_soc_dai *dai, + struct meson_codec_glue_input *data) +{ + dai->playback_dma_data = data; +} + +struct meson_codec_glue_input * +meson_codec_glue_input_get_data(struct snd_soc_dai *dai) +{ + return dai->playback_dma_data; +} +EXPORT_SYMBOL_GPL(meson_codec_glue_input_get_data); + +static struct meson_codec_glue_input * +meson_codec_glue_output_get_input_data(struct snd_soc_dapm_widget *w) +{ + struct snd_soc_dapm_widget *in = + meson_codec_glue_get_input(w); + struct snd_soc_dai *dai; + + if (WARN_ON(!in)) + return NULL; + + dai = in->priv; + + return meson_codec_glue_input_get_data(dai); +} + +int meson_codec_glue_input_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct meson_codec_glue_input *data = + meson_codec_glue_input_get_data(dai); + + data->params.rates = snd_pcm_rate_to_rate_bit(params_rate(params)); + data->params.rate_min = params_rate(params); + data->params.rate_max = params_rate(params); + data->params.formats = 1ULL << (__force int) params_format(params); + data->params.channels_min = params_channels(params); + data->params.channels_max = params_channels(params); + data->params.sig_bits = dai->driver->playback.sig_bits; + + return 0; +} +EXPORT_SYMBOL_GPL(meson_codec_glue_input_hw_params); + +int meson_codec_glue_input_set_fmt(struct snd_soc_dai *dai, + unsigned int fmt) +{ + struct meson_codec_glue_input *data = + meson_codec_glue_input_get_data(dai); + + /* Save the source stream format for the downstream link */ + data->fmt = fmt; + return 0; +} +EXPORT_SYMBOL_GPL(meson_codec_glue_input_set_fmt); + +int meson_codec_glue_output_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct meson_codec_glue_input *in_data = + meson_codec_glue_output_get_input_data(dai->capture_widget); + + if (!in_data) + return -ENODEV; + + if (WARN_ON(!rtd->dai_link->params)) { + dev_warn(dai->dev, "codec2codec link expected\n"); + return -EINVAL; + } + + /* Replace link params with the input params */ + rtd->dai_link->params = &in_data->params; + + if (!in_data->fmt) + return 0; + + return snd_soc_runtime_set_dai_fmt(rtd, in_data->fmt); +} +EXPORT_SYMBOL_GPL(meson_codec_glue_output_startup); + +int meson_codec_glue_input_dai_probe(struct snd_soc_dai *dai) +{ + struct meson_codec_glue_input *data; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + meson_codec_glue_input_set_data(dai, data); + return 0; +} +EXPORT_SYMBOL_GPL(meson_codec_glue_input_dai_probe); + +int meson_codec_glue_input_dai_remove(struct snd_soc_dai *dai) +{ + struct meson_codec_glue_input *data = + meson_codec_glue_input_get_data(dai); + + kfree(data); + return 0; +} +EXPORT_SYMBOL_GPL(meson_codec_glue_input_dai_remove); + +MODULE_AUTHOR("Jerome Brunet "); +MODULE_DESCRIPTION("Amlogic Codec Glue Helpers"); +MODULE_LICENSE("GPL v2"); + diff --git a/sound/soc/meson/meson-codec-glue.h b/sound/soc/meson/meson-codec-glue.h new file mode 100644 index 000000000000..07f99446c0c6 --- /dev/null +++ b/sound/soc/meson/meson-codec-glue.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (c) 2018 Baylibre SAS. + * Author: Jerome Brunet + */ + +#ifndef _MESON_CODEC_GLUE_H +#define _MESON_CODEC_GLUE_H + +#include + +struct meson_codec_glue_input { + struct snd_soc_pcm_stream params; + unsigned int fmt; +}; + +/* Input helpers */ +struct meson_codec_glue_input * +meson_codec_glue_input_get_data(struct snd_soc_dai *dai); +int meson_codec_glue_input_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai); +int meson_codec_glue_input_set_fmt(struct snd_soc_dai *dai, + unsigned int fmt); +int meson_codec_glue_input_dai_probe(struct snd_soc_dai *dai); +int meson_codec_glue_input_dai_remove(struct snd_soc_dai *dai); + +/* Output helpers */ +int meson_codec_glue_output_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai); + +#endif /* _MESON_CODEC_GLUE_H */ diff --git a/sound/soc/meson/t9015.c b/sound/soc/meson/t9015.c new file mode 100644 index 000000000000..56d2592c16d5 --- /dev/null +++ b/sound/soc/meson/t9015.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2020 BayLibre, SAS. +// Author: Jerome Brunet + +#include +#include +#include +#include +#include +#include +#include +#include + +#define BLOCK_EN 0x00 +#define LORN_EN 0 +#define LORP_EN 1 +#define LOLN_EN 2 +#define LOLP_EN 3 +#define DACR_EN 4 +#define DACL_EN 5 +#define DACR_INV 20 +#define DACL_INV 21 +#define DACR_SRC 22 +#define DACL_SRC 23 +#define REFP_BUF_EN BIT(12) +#define BIAS_CURRENT_EN BIT(13) +#define VMID_GEN_FAST BIT(14) +#define VMID_GEN_EN BIT(15) +#define I2S_MODE BIT(30) +#define VOL_CTRL0 0x04 +#define GAIN_H 31 +#define GAIN_L 23 +#define VOL_CTRL1 0x08 +#define DAC_MONO 8 +#define RAMP_RATE 10 +#define VC_RAMP_MODE 12 +#define MUTE_MODE 13 +#define UNMUTE_MODE 14 +#define DAC_SOFT_MUTE 15 +#define DACR_VC 16 +#define DACL_VC 24 +#define LINEOUT_CFG 0x0c +#define LORN_POL 0 +#define LORP_POL 4 +#define LOLN_POL 8 +#define LOLP_POL 12 +#define POWER_CFG 0x10 + +struct t9015 { + struct clk *pclk; + struct regulator *avdd; +}; + +static int t9015_dai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) +{ + struct snd_soc_component *component = dai->component; + unsigned int val; + + switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { + case SND_SOC_DAIFMT_CBM_CFM: + val = I2S_MODE; + break; + + case SND_SOC_DAIFMT_CBS_CFS: + val = 0; + break; + + default: + return -EINVAL; + } + + snd_soc_component_update_bits(component, BLOCK_EN, I2S_MODE, val); + + if (((fmt & SND_SOC_DAIFMT_FORMAT_MASK) != SND_SOC_DAIFMT_I2S) && + ((fmt & SND_SOC_DAIFMT_FORMAT_MASK) != SND_SOC_DAIFMT_LEFT_J)) + return -EINVAL; + + return 0; +} + +static const struct snd_soc_dai_ops t9015_dai_ops = { + .set_fmt = t9015_dai_set_fmt, +}; + +static struct snd_soc_dai_driver t9015_dai = { + .name = "t9015-hifi", + .playback = { + .stream_name = "Playback", + .channels_min = 1, + .channels_max = 2, + .rates = SNDRV_PCM_RATE_8000_96000, + .formats = (SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_S16_LE | + SNDRV_PCM_FMTBIT_S20_LE | + SNDRV_PCM_FMTBIT_S24_LE), + }, + .ops = &t9015_dai_ops, +}; + +static const DECLARE_TLV_DB_MINMAX_MUTE(dac_vol_tlv, -9525, 0); + +static const char * const ramp_rate_txt[] = { "Fast", "Slow" }; +static SOC_ENUM_SINGLE_DECL(ramp_rate_enum, VOL_CTRL1, RAMP_RATE, + ramp_rate_txt); + +static const char * const dacr_in_txt[] = { "Right", "Left" }; +static SOC_ENUM_SINGLE_DECL(dacr_in_enum, BLOCK_EN, DACR_SRC, dacr_in_txt); + +static const char * const dacl_in_txt[] = { "Left", "Right" }; +static SOC_ENUM_SINGLE_DECL(dacl_in_enum, BLOCK_EN, DACL_SRC, dacl_in_txt); + +static const char * const mono_txt[] = { "Stereo", "Mono"}; +static SOC_ENUM_SINGLE_DECL(mono_enum, VOL_CTRL1, DAC_MONO, mono_txt); + +static const struct snd_kcontrol_new t9015_snd_controls[] = { + /* Volume Controls */ + SOC_ENUM("Playback Channel Mode", mono_enum), + SOC_SINGLE("Playback Switch", VOL_CTRL1, DAC_SOFT_MUTE, 1, 1), + SOC_DOUBLE_TLV("Playback Volume", VOL_CTRL1, DACL_VC, DACR_VC, + 0xff, 0, dac_vol_tlv), + + /* Ramp Controls */ + SOC_ENUM("Ramp Rate", ramp_rate_enum), + SOC_SINGLE("Volume Ramp Switch", VOL_CTRL1, VC_RAMP_MODE, 1, 0), + SOC_SINGLE("Mute Ramp Switch", VOL_CTRL1, MUTE_MODE, 1, 0), + SOC_SINGLE("Unmute Ramp Switch", VOL_CTRL1, UNMUTE_MODE, 1, 0), +}; + +static const struct snd_kcontrol_new t9015_right_dac_mux = + SOC_DAPM_ENUM("Right DAC Source", dacr_in_enum); +static const struct snd_kcontrol_new t9015_left_dac_mux = + SOC_DAPM_ENUM("Left DAC Source", dacl_in_enum); + +static const struct snd_soc_dapm_widget t9015_dapm_widgets[] = { + SND_SOC_DAPM_AIF_IN("Right IN", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_IN("Left IN", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_MUX("Right DAC Sel", SND_SOC_NOPM, 0, 0, + &t9015_right_dac_mux), + SND_SOC_DAPM_MUX("Left DAC Sel", SND_SOC_NOPM, 0, 0, + &t9015_left_dac_mux), + SND_SOC_DAPM_DAC("Right DAC", NULL, BLOCK_EN, DACR_EN, 0), + SND_SOC_DAPM_DAC("Left DAC", NULL, BLOCK_EN, DACL_EN, 0), + SND_SOC_DAPM_OUT_DRV("Right- Driver", BLOCK_EN, LORN_EN, 0, + NULL, 0), + SND_SOC_DAPM_OUT_DRV("Right+ Driver", BLOCK_EN, LORP_EN, 0, + NULL, 0), + SND_SOC_DAPM_OUT_DRV("Left- Driver", BLOCK_EN, LOLN_EN, 0, + NULL, 0), + SND_SOC_DAPM_OUT_DRV("Left+ Driver", BLOCK_EN, LOLP_EN, 0, + NULL, 0), + SND_SOC_DAPM_OUTPUT("LORN"), + SND_SOC_DAPM_OUTPUT("LORP"), + SND_SOC_DAPM_OUTPUT("LOLN"), + SND_SOC_DAPM_OUTPUT("LOLP"), +}; + +static const struct snd_soc_dapm_route t9015_dapm_routes[] = { + { "Right IN", NULL, "Playback" }, + { "Left IN", NULL, "Playback" }, + { "Right DAC Sel", "Right", "Right IN" }, + { "Right DAC Sel", "Left", "Left IN" }, + { "Left DAC Sel", "Right", "Right IN" }, + { "Left DAC Sel", "Left", "Left IN" }, + { "Right DAC", NULL, "Right DAC Sel" }, + { "Left DAC", NULL, "Left DAC Sel" }, + { "Right- Driver", NULL, "Right DAC" }, + { "Right+ Driver", NULL, "Right DAC" }, + { "Left- Driver", NULL, "Left DAC" }, + { "Left+ Driver", NULL, "Left DAC" }, + { "LORN", NULL, "Right- Driver", }, + { "LORP", NULL, "Right+ Driver", }, + { "LOLN", NULL, "Left- Driver", }, + { "LOLP", NULL, "Left+ Driver", }, +}; + +static int t9015_set_bias_level(struct snd_soc_component *component, + enum snd_soc_bias_level level) +{ + struct t9015 *priv = snd_soc_component_get_drvdata(component); + enum snd_soc_bias_level now = + snd_soc_component_get_bias_level(component); + int ret; + + switch (level) { + case SND_SOC_BIAS_ON: + snd_soc_component_update_bits(component, BLOCK_EN, + BIAS_CURRENT_EN, + BIAS_CURRENT_EN); + break; + case SND_SOC_BIAS_PREPARE: + snd_soc_component_update_bits(component, BLOCK_EN, + BIAS_CURRENT_EN, + 0); + break; + case SND_SOC_BIAS_STANDBY: + ret = regulator_enable(priv->avdd); + if (ret) { + dev_err(component->dev, "AVDD enable failed\n"); + return ret; + } + + if (now == SND_SOC_BIAS_OFF) { + snd_soc_component_update_bits(component, BLOCK_EN, + VMID_GEN_EN | VMID_GEN_FAST | REFP_BUF_EN, + VMID_GEN_EN | VMID_GEN_FAST | REFP_BUF_EN); + + mdelay(200); + snd_soc_component_update_bits(component, BLOCK_EN, + VMID_GEN_FAST, + 0); + } + + break; + case SND_SOC_BIAS_OFF: + snd_soc_component_update_bits(component, BLOCK_EN, + VMID_GEN_EN | VMID_GEN_FAST | REFP_BUF_EN, + 0); + + regulator_disable(priv->avdd); + break; + } + + return 0; +} + +static const struct snd_soc_component_driver t9015_codec_driver = { + .set_bias_level = t9015_set_bias_level, + .controls = t9015_snd_controls, + .num_controls = ARRAY_SIZE(t9015_snd_controls), + .dapm_widgets = t9015_dapm_widgets, + .num_dapm_widgets = ARRAY_SIZE(t9015_dapm_widgets), + .dapm_routes = t9015_dapm_routes, + .num_dapm_routes = ARRAY_SIZE(t9015_dapm_routes), + .suspend_bias_off = 1, + .endianness = 1, + .non_legacy_dai_naming = 1, +}; + +static const struct regmap_config t9015_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = POWER_CFG, +}; + +static int t9015_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct t9015 *priv; + void __iomem *regs; + struct regmap *regmap; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + platform_set_drvdata(pdev, priv); + + priv->pclk = devm_clk_get(dev, "pclk"); + if (IS_ERR(priv->pclk)) { + if (PTR_ERR(priv->pclk) != -EPROBE_DEFER) + dev_err(dev, "failed to get core clock\n"); + return PTR_ERR(priv->pclk); + } + + priv->avdd = devm_regulator_get(dev, "AVDD"); + if (IS_ERR(priv->avdd)) { + if (PTR_ERR(priv->avdd) != -EPROBE_DEFER) + dev_err(dev, "failed to AVDD\n"); + return PTR_ERR(priv->avdd); + } + + ret = clk_prepare_enable(priv->pclk); + if (ret) { + dev_err(dev, "core clock enable failed\n"); + return ret; + } + + ret = devm_add_action_or_reset(dev, + (void(*)(void *))clk_disable_unprepare, + priv->pclk); + if (ret) + return ret; + + ret = device_reset(dev); + if (ret) { + dev_err(dev, "reset failed\n"); + return ret; + } + + regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(regs)) { + dev_err(dev, "register map failed\n"); + return PTR_ERR(regs); + } + + regmap = devm_regmap_init_mmio(dev, regs, &t9015_regmap_config); + if (IS_ERR(regmap)) { + dev_err(dev, "regmap init failed\n"); + return PTR_ERR(regmap); + } + + /* + * Initialize output polarity: + * ATM the output polarity is fixed but in the future it might useful + * to add DT property to set this depending on the platform needs + */ + regmap_write(regmap, LINEOUT_CFG, 0x1111); + + return devm_snd_soc_register_component(dev, &t9015_codec_driver, + &t9015_dai, 1); +} + +static const struct of_device_id t9015_ids[] = { + { .compatible = "amlogic,t9015", }, + { } +}; +MODULE_DEVICE_TABLE(of, t9015_ids); + +static struct platform_driver t9015_driver = { + .driver = { + .name = "t9015-codec", + .of_match_table = of_match_ptr(t9015_ids), + }, + .probe = t9015_probe, +}; + +module_platform_driver(t9015_driver); + +MODULE_DESCRIPTION("ASoC Amlogic T9015 codec driver"); +MODULE_AUTHOR("Jerome Brunet "); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/mxs/mxs-sgtl5000.c b/sound/soc/mxs/mxs-sgtl5000.c index 9841e1da9782..f46d7aca8cf6 100644 --- a/sound/soc/mxs/mxs-sgtl5000.c +++ b/sound/soc/mxs/mxs-sgtl5000.c @@ -20,8 +20,8 @@ static int mxs_sgtl5000_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int rate = params_rate(params); u32 mclk; int ret; diff --git a/sound/soc/pxa/Kconfig b/sound/soc/pxa/Kconfig index 295cfffa4646..d4c0f580a565 100644 --- a/sound/soc/pxa/Kconfig +++ b/sound/soc/pxa/Kconfig @@ -81,6 +81,9 @@ config SND_PXA2XX_SOC_TOSA depends on SND_PXA2XX_SOC && MACH_TOSA depends on MFD_TC6393XB depends on AC97_BUS=n + select REGMAP + select AC97_BUS_NEW + select AC97_BUS_COMPAT select SND_PXA2XX_SOC_AC97 select SND_SOC_WM9712 help @@ -91,6 +94,9 @@ config SND_PXA2XX_SOC_E740 tristate "SoC AC97 Audio support for e740" depends on SND_PXA2XX_SOC && MACH_E740 depends on AC97_BUS=n + select REGMAP + select AC97_BUS_NEW + select AC97_BUS_COMPAT select SND_SOC_WM9705 select SND_PXA2XX_SOC_AC97 help @@ -101,6 +107,7 @@ config SND_PXA2XX_SOC_E750 tristate "SoC AC97 Audio support for e750" depends on SND_PXA2XX_SOC && MACH_E750 depends on AC97_BUS=n + select REGMAP select SND_SOC_WM9705 select SND_PXA2XX_SOC_AC97 help @@ -111,7 +118,10 @@ config SND_PXA2XX_SOC_E800 tristate "SoC AC97 Audio support for e800" depends on SND_PXA2XX_SOC && MACH_E800 depends on AC97_BUS=n + select REGMAP select SND_SOC_WM9712 + select AC97_BUS_NEW + select AC97_BUS_COMPAT select SND_PXA2XX_SOC_AC97 help Say Y if you want to add support for SoC audio on the @@ -122,6 +132,9 @@ config SND_PXA2XX_SOC_EM_X270 depends on SND_PXA2XX_SOC && (MACH_EM_X270 || MACH_EXEDA || \ MACH_CM_X300) depends on AC97_BUS=n + select REGMAP + select AC97_BUS_NEW + select AC97_BUS_COMPAT select SND_PXA2XX_SOC_AC97 select SND_SOC_WM9712 help @@ -133,6 +146,9 @@ config SND_PXA2XX_SOC_PALM27X depends on SND_PXA2XX_SOC && (MACH_PALMLD || MACH_PALMTX || \ MACH_PALMT5 || MACH_PALMTE2) depends on AC97_BUS=n + select REGMAP + select AC97_BUS_NEW + select AC97_BUS_COMPAT select SND_PXA2XX_SOC_AC97 select SND_SOC_WM9712 help @@ -163,7 +179,10 @@ config SND_SOC_ZYLONITE tristate "SoC Audio support for Marvell Zylonite" depends on SND_PXA2XX_SOC && MACH_ZYLONITE depends on AC97_BUS=n + select AC97_BUS_NEW + select AC97_BUS_COMPAT select SND_PXA2XX_SOC_AC97 + select REGMAP select SND_PXA_SOC_SSP select SND_SOC_WM9713 help @@ -193,6 +212,9 @@ config SND_PXA2XX_SOC_MIOA701 tristate "SoC Audio support for MIO A701" depends on SND_PXA2XX_SOC && MACH_MIOA701 depends on AC97_BUS=n + select REGMAP + select AC97_BUS_NEW + select AC97_BUS_COMPAT select SND_PXA2XX_SOC_AC97 select SND_SOC_WM9713 help diff --git a/sound/soc/pxa/brownstone.c b/sound/soc/pxa/brownstone.c index 53b1435ced3f..016a91199485 100644 --- a/sound/soc/pxa/brownstone.c +++ b/sound/soc/pxa/brownstone.c @@ -44,8 +44,8 @@ static int brownstone_wm8994_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int freq_out, sspa_mclk, sysclk; if (params_rate(params) > 11025) { diff --git a/sound/soc/pxa/corgi.c b/sound/soc/pxa/corgi.c index d81082323fb4..6fbef9a0afa7 100644 --- a/sound/soc/pxa/corgi.c +++ b/sound/soc/pxa/corgi.c @@ -116,8 +116,8 @@ static int corgi_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int clk = 0; int ret = 0; diff --git a/sound/soc/pxa/hx4700.c b/sound/soc/pxa/hx4700.c index 0139343dbcce..b4da9a9a6521 100644 --- a/sound/soc/pxa/hx4700.c +++ b/sound/soc/pxa/hx4700.c @@ -54,8 +54,8 @@ static int hx4700_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int ret = 0; /* set the I2S system clock as output */ diff --git a/sound/soc/pxa/imote2.c b/sound/soc/pxa/imote2.c index 514e17724fc3..3014e8244ab4 100644 --- a/sound/soc/pxa/imote2.c +++ b/sound/soc/pxa/imote2.c @@ -12,8 +12,8 @@ static int imote2_asoc_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int clk = 0; int ret; diff --git a/sound/soc/pxa/magician.c b/sound/soc/pxa/magician.c index 6483cff5b73d..e4c818f4cd62 100644 --- a/sound/soc/pxa/magician.c +++ b/sound/soc/pxa/magician.c @@ -83,8 +83,8 @@ static int magician_playback_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int width; int ret = 0; @@ -121,8 +121,8 @@ static int magician_capture_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int ret = 0; /* set codec DAI configuration */ @@ -358,10 +358,10 @@ static int __init magician_init(void) adapter = i2c_get_adapter(0); if (!adapter) return -ENODEV; - client = i2c_new_device(adapter, i2c_board_info); + client = i2c_new_client_device(adapter, i2c_board_info); i2c_put_adapter(adapter); - if (!client) - return -ENODEV; + if (IS_ERR(client)) + return PTR_ERR(client); ret = gpio_request(EGPIO_MAGICIAN_SPK_POWER, "SPK_POWER"); if (ret) diff --git a/sound/soc/pxa/mioa701_wm9713.c b/sound/soc/pxa/mioa701_wm9713.c index 76e054d514a8..bf27b277c01f 100644 --- a/sound/soc/pxa/mioa701_wm9713.c +++ b/sound/soc/pxa/mioa701_wm9713.c @@ -73,7 +73,7 @@ static int rear_amp_event(struct snd_soc_dapm_widget *widget, struct snd_soc_component *component; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - component = rtd->codec_dai->component; + component = asoc_rtd_to_codec(rtd, 0)->component; return rear_amp_power(component, SND_SOC_DAPM_EVENT_ON(event)); } @@ -117,7 +117,7 @@ static const struct snd_soc_dapm_route audio_map[] = { static int mioa701_wm9713_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; /* Prepare GPIO8 for rear speaker amplifier */ snd_soc_component_update_bits(component, AC97_GPIO_CFG, 0x100, 0x100); diff --git a/sound/soc/pxa/mmp-pcm.c b/sound/soc/pxa/mmp-pcm.c index 287b5da739e5..3fe6c4c5a3ab 100644 --- a/sound/soc/pxa/mmp-pcm.c +++ b/sound/soc/pxa/mmp-pcm.c @@ -112,7 +112,7 @@ static int mmp_pcm_open(struct snd_soc_component *component, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct platform_device *pdev = to_platform_device(component->dev); - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct mmp_dma_data dma_data; struct resource *r; diff --git a/sound/soc/pxa/mmp-sspa.c b/sound/soc/pxa/mmp-sspa.c index e701637a9ae9..3548a2634a63 100644 --- a/sound/soc/pxa/mmp-sspa.c +++ b/sound/soc/pxa/mmp-sspa.c @@ -251,7 +251,7 @@ static int mmp_sspa_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct sspa_priv *sspa_priv = snd_soc_dai_get_drvdata(dai); struct ssp_device *sspa = sspa_priv->sspa; struct snd_dmaengine_dai_dma_data *dma_params; diff --git a/sound/soc/pxa/poodle.c b/sound/soc/pxa/poodle.c index 59ef04d0467a..287984a564c8 100644 --- a/sound/soc/pxa/poodle.c +++ b/sound/soc/pxa/poodle.c @@ -90,8 +90,8 @@ static int poodle_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int clk = 0; int ret = 0; diff --git a/sound/soc/pxa/pxa2xx-i2s.c b/sound/soc/pxa/pxa2xx-i2s.c index 5f1c477b5833..9a32bf72127a 100644 --- a/sound/soc/pxa/pxa2xx-i2s.c +++ b/sound/soc/pxa/pxa2xx-i2s.c @@ -96,7 +96,7 @@ static int pxa2xx_i2s_startup(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); if (IS_ERR(clk_i2s)) return PTR_ERR(clk_i2s); diff --git a/sound/soc/pxa/spitz.c b/sound/soc/pxa/spitz.c index f7babffb7228..6d8174f62935 100644 --- a/sound/soc/pxa/spitz.c +++ b/sound/soc/pxa/spitz.c @@ -117,8 +117,8 @@ static int spitz_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int clk = 0; int ret = 0; diff --git a/sound/soc/pxa/ttc-dkb.c b/sound/soc/pxa/ttc-dkb.c index d8f79e2266b1..d5f2961b1a3e 100644 --- a/sound/soc/pxa/ttc-dkb.c +++ b/sound/soc/pxa/ttc-dkb.c @@ -61,7 +61,7 @@ static const struct snd_soc_dapm_route ttc_audio_map[] = { static int ttc_pm860x_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; /* Headset jack detection */ snd_soc_card_jack_new(rtd->card, "Headphone Jack", SND_JACK_HEADPHONE | diff --git a/sound/soc/pxa/z2.c b/sound/soc/pxa/z2.c index f9a33cb36f5b..6eee1aefc89a 100644 --- a/sound/soc/pxa/z2.c +++ b/sound/soc/pxa/z2.c @@ -34,8 +34,8 @@ static int z2_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int clk = 0; int ret = 0; diff --git a/sound/soc/pxa/zylonite.c b/sound/soc/pxa/zylonite.c index 567dc133ea92..447b59b8bd33 100644 --- a/sound/soc/pxa/zylonite.c +++ b/sound/soc/pxa/zylonite.c @@ -66,7 +66,7 @@ static const struct snd_soc_dapm_route audio_map[] = { static int zylonite_wm9713_init(struct snd_soc_pcm_runtime *rtd) { if (clk_pout) - snd_soc_dai_set_pll(rtd->codec_dai, 0, 0, + snd_soc_dai_set_pll(asoc_rtd_to_codec(rtd, 0), 0, 0, clk_get_rate(pout), 0); return 0; @@ -76,8 +76,8 @@ static int zylonite_voice_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int wm9713_div = 0; int ret = 0; int rate = params_rate(params); diff --git a/sound/soc/qcom/Kconfig b/sound/soc/qcom/Kconfig index 6530d2462a9e..f51b28d1b94d 100644 --- a/sound/soc/qcom/Kconfig +++ b/sound/soc/qcom/Kconfig @@ -99,7 +99,7 @@ config SND_SOC_MSM8996 config SND_SOC_SDM845 tristate "SoC Machine driver for SDM845 boards" - depends on QCOM_APR && CROS_EC && I2C + depends on QCOM_APR && CROS_EC && I2C && SOUNDWIRE select SND_SOC_QDSP6 select SND_SOC_QCOM_COMMON select SND_SOC_RT5663 diff --git a/sound/soc/qcom/apq8016_sbc.c b/sound/soc/qcom/apq8016_sbc.c index ac75838bbfab..2ef090f4af9e 100644 --- a/sound/soc/qcom/apq8016_sbc.c +++ b/sound/soc/qcom/apq8016_sbc.c @@ -33,9 +33,9 @@ struct apq8016_sbc_data { static int apq8016_sbc_dai_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai; struct snd_soc_component *component; - struct snd_soc_dai_link *dai_link = rtd->dai_link; struct snd_soc_card *card = rtd->card; struct apq8016_sbc_data *pdata = snd_soc_card_get_drvdata(card); int i, rval; @@ -90,10 +90,9 @@ static int apq8016_sbc_dai_init(struct snd_soc_pcm_runtime *rtd) pdata->jack_setup = true; } - for (i = 0 ; i < dai_link->num_codecs; i++) { - struct snd_soc_dai *dai = rtd->codec_dais[i]; + for_each_rtd_codec_dais(rtd, i, codec_dai) { - component = dai->component; + component = codec_dai->component; /* Set default mclk for internal codec */ rval = snd_soc_component_set_sysclk(component, 0, 0, DEFAULT_MCLK_RATE, SND_SOC_CLOCK_IN); diff --git a/sound/soc/qcom/apq8096.c b/sound/soc/qcom/apq8096.c index 94363fd6846a..d55e3ad96716 100644 --- a/sound/soc/qcom/apq8096.c +++ b/sound/soc/qcom/apq8096.c @@ -31,8 +31,8 @@ static int msm_snd_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); u32 rx_ch[SLIM_MAX_RX_PORTS], tx_ch[SLIM_MAX_TX_PORTS]; u32 rx_ch_cnt = 0, tx_ch_cnt = 0; int ret = 0; @@ -66,7 +66,7 @@ static struct snd_soc_ops apq8096_ops = { static int apq8096_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); /* * Codec SLIMBUS configuration diff --git a/sound/soc/qcom/lpass-platform.c b/sound/soc/qcom/lpass-platform.c index b05091c283b7..34f7fd1bab1c 100644 --- a/sound/soc/qcom/lpass-platform.c +++ b/sound/soc/qcom/lpass-platform.c @@ -55,7 +55,7 @@ static int lpass_platform_pcmops_open(struct snd_soc_component *component, { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *soc_runtime = substream->private_data; - struct snd_soc_dai *cpu_dai = soc_runtime->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(soc_runtime, 0); struct lpass_data *drvdata = snd_soc_component_get_drvdata(component); struct lpass_variant *v = drvdata->variant; int ret, dma_ch, dir = substream->stream; @@ -529,7 +529,7 @@ static void lpass_platform_pcm_free(struct snd_soc_component *component, struct snd_pcm_substream *substream; int i; - for (i = 0; i < ARRAY_SIZE(pcm->streams); i++) { + for_each_pcm_streams(i) { substream = pcm->streams[i].substream; if (substream) { snd_dma_free_pages(&substream->dma_buffer); diff --git a/sound/soc/qcom/qdsp6/q6asm-dai.c b/sound/soc/qcom/qdsp6/q6asm-dai.c index c0d422d0ab94..f6c7cddf08e8 100644 --- a/sound/soc/qcom/qdsp6/q6asm-dai.c +++ b/sound/soc/qcom/qdsp6/q6asm-dai.c @@ -41,6 +41,9 @@ #define Q6ASM_DAI_TX 1 #define Q6ASM_DAI_RX 2 +#define ALAC_CH_LAYOUT_MONO ((101 << 16) | 1) +#define ALAC_CH_LAYOUT_STEREO ((101 << 16) | 2) + enum stream_state { Q6ASM_STREAM_IDLE = 0, Q6ASM_STREAM_STOPPED, @@ -69,6 +72,8 @@ struct q6asm_dai_rtd { }; struct q6asm_dai_data { + struct snd_soc_dai_driver *dais; + int num_dais; long long int sid; }; @@ -250,7 +255,7 @@ static int q6asm_dai_prepare(struct snd_soc_component *component, if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { ret = q6asm_open_write(prtd->audio_client, FORMAT_LINEAR_PCM, - prtd->bits_per_sample); + 0, prtd->bits_per_sample); } else if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) { ret = q6asm_open_read(prtd->audio_client, FORMAT_LINEAR_PCM, prtd->bits_per_sample); @@ -328,7 +333,7 @@ static int q6asm_dai_open(struct snd_soc_component *component, { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *soc_prtd = substream->private_data; - struct snd_soc_dai *cpu_dai = soc_prtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(soc_prtd, 0); struct q6asm_dai_rtd *prtd; struct q6asm_dai_data *pdata; struct device *dev = component->dev; @@ -540,7 +545,7 @@ static int q6asm_dai_compr_open(struct snd_compr_stream *stream) struct snd_soc_pcm_runtime *rtd = stream->private_data; struct snd_soc_component *c = snd_soc_rtdcom_lookup(rtd, DRV_NAME); struct snd_compr_runtime *runtime = stream->runtime; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct q6asm_dai_data *pdata; struct device *dev = c->dev; struct q6asm_dai_rtd *prtd; @@ -627,10 +632,17 @@ static int q6asm_dai_compr_set_params(struct snd_compr_stream *stream, int dir = stream->direction; struct q6asm_dai_data *pdata; struct q6asm_flac_cfg flac_cfg; + struct q6asm_wma_cfg wma_cfg; + struct q6asm_alac_cfg alac_cfg; + struct q6asm_ape_cfg ape_cfg; + unsigned int wma_v9 = 0; struct device *dev = c->dev; int ret; union snd_codec_options *codec_options; struct snd_dec_flac *flac; + struct snd_dec_wma *wma; + struct snd_dec_alac *alac; + struct snd_dec_ape *ape; codec_options = &(prtd->codec_param.codec.options); @@ -652,7 +664,7 @@ static int q6asm_dai_compr_set_params(struct snd_compr_stream *stream, prtd->bits_per_sample = 16; if (dir == SND_COMPRESS_PLAYBACK) { ret = q6asm_open_write(prtd->audio_client, params->codec.id, - prtd->bits_per_sample); + params->codec.profile, prtd->bits_per_sample); if (ret < 0) { dev_err(dev, "q6asm_open_write failed\n"); @@ -692,6 +704,126 @@ static int q6asm_dai_compr_set_params(struct snd_compr_stream *stream, return -EIO; } break; + + case SND_AUDIOCODEC_WMA: + wma = &codec_options->wma_d; + + memset(&wma_cfg, 0x0, sizeof(struct q6asm_wma_cfg)); + + wma_cfg.sample_rate = params->codec.sample_rate; + wma_cfg.num_channels = params->codec.ch_in; + wma_cfg.bytes_per_sec = params->codec.bit_rate / 8; + wma_cfg.block_align = params->codec.align; + wma_cfg.bits_per_sample = prtd->bits_per_sample; + wma_cfg.enc_options = wma->encoder_option; + wma_cfg.adv_enc_options = wma->adv_encoder_option; + wma_cfg.adv_enc_options2 = wma->adv_encoder_option2; + + if (wma_cfg.num_channels == 1) + wma_cfg.channel_mask = 4; /* Mono Center */ + else if (wma_cfg.num_channels == 2) + wma_cfg.channel_mask = 3; /* Stereo FL/FR */ + else + return -EINVAL; + + /* check the codec profile */ + switch (params->codec.profile) { + case SND_AUDIOPROFILE_WMA9: + wma_cfg.fmtag = 0x161; + wma_v9 = 1; + break; + + case SND_AUDIOPROFILE_WMA10: + wma_cfg.fmtag = 0x166; + break; + + case SND_AUDIOPROFILE_WMA9_PRO: + wma_cfg.fmtag = 0x162; + break; + + case SND_AUDIOPROFILE_WMA9_LOSSLESS: + wma_cfg.fmtag = 0x163; + break; + + case SND_AUDIOPROFILE_WMA10_LOSSLESS: + wma_cfg.fmtag = 0x167; + break; + + default: + dev_err(dev, "Unknown WMA profile:%x\n", + params->codec.profile); + return -EIO; + } + + if (wma_v9) + ret = q6asm_stream_media_format_block_wma_v9( + prtd->audio_client, &wma_cfg); + else + ret = q6asm_stream_media_format_block_wma_v10( + prtd->audio_client, &wma_cfg); + if (ret < 0) { + dev_err(dev, "WMA9 CMD failed:%d\n", ret); + return -EIO; + } + break; + + case SND_AUDIOCODEC_ALAC: + memset(&alac_cfg, 0x0, sizeof(alac_cfg)); + alac = &codec_options->alac_d; + + alac_cfg.sample_rate = params->codec.sample_rate; + alac_cfg.avg_bit_rate = params->codec.bit_rate; + alac_cfg.bit_depth = prtd->bits_per_sample; + alac_cfg.num_channels = params->codec.ch_in; + + alac_cfg.frame_length = alac->frame_length; + alac_cfg.pb = alac->pb; + alac_cfg.mb = alac->mb; + alac_cfg.kb = alac->kb; + alac_cfg.max_run = alac->max_run; + alac_cfg.compatible_version = alac->compatible_version; + alac_cfg.max_frame_bytes = alac->max_frame_bytes; + + switch (params->codec.ch_in) { + case 1: + alac_cfg.channel_layout_tag = ALAC_CH_LAYOUT_MONO; + break; + case 2: + alac_cfg.channel_layout_tag = ALAC_CH_LAYOUT_STEREO; + break; + } + ret = q6asm_stream_media_format_block_alac(prtd->audio_client, + &alac_cfg); + if (ret < 0) { + dev_err(dev, "ALAC CMD Format block failed:%d\n", ret); + return -EIO; + } + break; + + case SND_AUDIOCODEC_APE: + memset(&ape_cfg, 0x0, sizeof(ape_cfg)); + ape = &codec_options->ape_d; + + ape_cfg.sample_rate = params->codec.sample_rate; + ape_cfg.num_channels = params->codec.ch_in; + ape_cfg.bits_per_sample = prtd->bits_per_sample; + + ape_cfg.compatible_version = ape->compatible_version; + ape_cfg.compression_level = ape->compression_level; + ape_cfg.format_flags = ape->format_flags; + ape_cfg.blocks_per_frame = ape->blocks_per_frame; + ape_cfg.final_frame_blocks = ape->final_frame_blocks; + ape_cfg.total_frames = ape->total_frames; + ape_cfg.seek_table_present = ape->seek_table_present; + + ret = q6asm_stream_media_format_block_ape(prtd->audio_client, + &ape_cfg); + if (ret < 0) { + dev_err(dev, "APE CMD Format block failed:%d\n", ret); + return -EIO; + } + break; + default: break; } @@ -791,9 +923,12 @@ static int q6asm_dai_compr_get_caps(struct snd_compr_stream *stream, caps->max_fragment_size = COMPR_PLAYBACK_MAX_FRAGMENT_SIZE; caps->min_fragments = COMPR_PLAYBACK_MIN_NUM_FRAGMENTS; caps->max_fragments = COMPR_PLAYBACK_MAX_NUM_FRAGMENTS; - caps->num_codecs = 2; + caps->num_codecs = 5; caps->codecs[0] = SND_AUDIOCODEC_MP3; caps->codecs[1] = SND_AUDIOCODEC_FLAC; + caps->codecs[2] = SND_AUDIOCODEC_WMA; + caps->codecs[3] = SND_AUDIOCODEC_ALAC; + caps->codecs[4] = SND_AUDIOCODEC_APE; return 0; } @@ -889,7 +1024,7 @@ static const struct snd_soc_component_driver q6asm_fe_dai_component = { .compr_ops = &q6asm_dai_compr_ops, }; -static struct snd_soc_dai_driver q6asm_fe_dais[] = { +static struct snd_soc_dai_driver q6asm_fe_dais_template[] = { Q6ASM_FEDAI_DRIVER(1), Q6ASM_FEDAI_DRIVER(2), Q6ASM_FEDAI_DRIVER(3), @@ -903,10 +1038,22 @@ static struct snd_soc_dai_driver q6asm_fe_dais[] = { static int of_q6asm_parse_dai_data(struct device *dev, struct q6asm_dai_data *pdata) { - static struct snd_soc_dai_driver *dai_drv; + struct snd_soc_dai_driver *dai_drv; struct snd_soc_pcm_stream empty_stream; struct device_node *node; - int ret, id, dir; + int ret, id, dir, idx = 0; + + + pdata->num_dais = of_get_child_count(dev->of_node); + if (!pdata->num_dais) { + dev_err(dev, "No dais found in DT\n"); + return -EINVAL; + } + + pdata->dais = devm_kcalloc(dev, pdata->num_dais, sizeof(*dai_drv), + GFP_KERNEL); + if (!pdata->dais) + return -ENOMEM; memset(&empty_stream, 0, sizeof(empty_stream)); @@ -917,7 +1064,8 @@ static int of_q6asm_parse_dai_data(struct device *dev, continue; } - dai_drv = &q6asm_fe_dais[id]; + dai_drv = &pdata->dais[idx++]; + *dai_drv = q6asm_fe_dais_template[id]; ret = of_property_read_u32(node, "direction", &dir); if (ret) @@ -955,11 +1103,12 @@ static int q6asm_dai_probe(struct platform_device *pdev) dev_set_drvdata(dev, pdata); - of_q6asm_parse_dai_data(dev, pdata); + rc = of_q6asm_parse_dai_data(dev, pdata); + if (rc) + return rc; return devm_snd_soc_register_component(dev, &q6asm_fe_dai_component, - q6asm_fe_dais, - ARRAY_SIZE(q6asm_fe_dais)); + pdata->dais, pdata->num_dais); } static const struct of_device_id q6asm_dai_device_id[] = { diff --git a/sound/soc/qcom/qdsp6/q6asm.c b/sound/soc/qcom/qdsp6/q6asm.c index 36e0eab13a98..0e0e8f7a460a 100644 --- a/sound/soc/qcom/qdsp6/q6asm.c +++ b/sound/soc/qcom/qdsp6/q6asm.c @@ -39,6 +39,8 @@ #define ASM_MEDIA_FMT_MULTI_CHANNEL_PCM_V2 0x00010DA5 #define ASM_MEDIA_FMT_MP3 0x00010BE9 #define ASM_MEDIA_FMT_FLAC 0x00010C16 +#define ASM_MEDIA_FMT_WMA_V9 0x00010DA8 +#define ASM_MEDIA_FMT_WMA_V10 0x00010DA7 #define ASM_DATA_CMD_WRITE_V2 0x00010DAB #define ASM_DATA_CMD_READ_V2 0x00010DAC #define ASM_SESSION_CMD_SUSPEND 0x00010DEC @@ -46,6 +48,8 @@ #define ASM_STREAM_CMD_OPEN_READ_V3 0x00010DB4 #define ASM_DATA_EVENT_READ_DONE_V2 0x00010D9A #define ASM_STREAM_CMD_OPEN_READWRITE_V2 0x00010D8D +#define ASM_MEDIA_FMT_ALAC 0x00012f31 +#define ASM_MEDIA_FMT_APE 0x00012f32 #define ASM_LEGACY_STREAM_SESSION 0 @@ -104,6 +108,63 @@ struct asm_flac_fmt_blk_v2 { u16 reserved; } __packed; +struct asm_wmastdv9_fmt_blk_v2 { + struct asm_data_cmd_media_fmt_update_v2 fmt_blk; + u16 fmtag; + u16 num_channels; + u32 sample_rate; + u32 bytes_per_sec; + u16 blk_align; + u16 bits_per_sample; + u32 channel_mask; + u16 enc_options; + u16 reserved; +} __packed; + +struct asm_wmaprov10_fmt_blk_v2 { + struct asm_data_cmd_media_fmt_update_v2 fmt_blk; + u16 fmtag; + u16 num_channels; + u32 sample_rate; + u32 bytes_per_sec; + u16 blk_align; + u16 bits_per_sample; + u32 channel_mask; + u16 enc_options; + u16 advanced_enc_options1; + u32 advanced_enc_options2; +} __packed; + +struct asm_alac_fmt_blk_v2 { + struct asm_data_cmd_media_fmt_update_v2 fmt_blk; + u32 frame_length; + u8 compatible_version; + u8 bit_depth; + u8 pb; + u8 mb; + u8 kb; + u8 num_channels; + u16 max_run; + u32 max_frame_bytes; + u32 avg_bit_rate; + u32 sample_rate; + u32 channel_layout_tag; +} __packed; + +struct asm_ape_fmt_blk_v2 { + struct asm_data_cmd_media_fmt_update_v2 fmt_blk; + u16 compatible_version; + u16 compression_level; + u32 format_flags; + u32 blocks_per_frame; + u32 final_frame_blocks; + u32 total_frames; + u16 bits_per_sample; + u16 num_channels; + u32 sample_rate; + u32 seek_table_present; +} __packed; + struct asm_stream_cmd_set_encdec_param { u32 param_id; u32 param_size; @@ -858,7 +919,7 @@ static int q6asm_ac_send_cmd_sync(struct audio_client *ac, struct apr_pkt *pkt) * Return: Will be an negative value on error or zero on success */ int q6asm_open_write(struct audio_client *ac, uint32_t format, - uint16_t bits_per_sample) + u32 codec_profile, uint16_t bits_per_sample) { struct asm_stream_cmd_open_write_v3 *open; struct apr_pkt *pkt; @@ -894,6 +955,30 @@ int q6asm_open_write(struct audio_client *ac, uint32_t format, case SND_AUDIOCODEC_FLAC: open->dec_fmt_id = ASM_MEDIA_FMT_FLAC; break; + case SND_AUDIOCODEC_WMA: + switch (codec_profile) { + case SND_AUDIOPROFILE_WMA9: + open->dec_fmt_id = ASM_MEDIA_FMT_WMA_V9; + break; + case SND_AUDIOPROFILE_WMA10: + case SND_AUDIOPROFILE_WMA9_PRO: + case SND_AUDIOPROFILE_WMA9_LOSSLESS: + case SND_AUDIOPROFILE_WMA10_LOSSLESS: + open->dec_fmt_id = ASM_MEDIA_FMT_WMA_V10; + break; + default: + dev_err(ac->dev, "Invalid codec profile 0x%x\n", + codec_profile); + rc = -EINVAL; + goto err; + } + break; + case SND_AUDIOCODEC_ALAC: + open->dec_fmt_id = ASM_MEDIA_FMT_ALAC; + break; + case SND_AUDIOCODEC_APE: + open->dec_fmt_id = ASM_MEDIA_FMT_APE; + break; default: dev_err(ac->dev, "Invalid format 0x%x\n", format); rc = -EINVAL; @@ -1075,6 +1160,162 @@ int q6asm_stream_media_format_block_flac(struct audio_client *ac, return rc; } EXPORT_SYMBOL_GPL(q6asm_stream_media_format_block_flac); + +int q6asm_stream_media_format_block_wma_v9(struct audio_client *ac, + struct q6asm_wma_cfg *cfg) +{ + struct asm_wmastdv9_fmt_blk_v2 *fmt; + struct apr_pkt *pkt; + void *p; + int rc, pkt_size; + + pkt_size = APR_HDR_SIZE + sizeof(*fmt); + p = kzalloc(pkt_size, GFP_KERNEL); + if (!p) + return -ENOMEM; + + pkt = p; + fmt = p + APR_HDR_SIZE; + + q6asm_add_hdr(ac, &pkt->hdr, pkt_size, true, ac->stream_id); + + pkt->hdr.opcode = ASM_DATA_CMD_MEDIA_FMT_UPDATE_V2; + fmt->fmt_blk.fmt_blk_size = sizeof(*fmt) - sizeof(fmt->fmt_blk); + fmt->fmtag = cfg->fmtag; + fmt->num_channels = cfg->num_channels; + fmt->sample_rate = cfg->sample_rate; + fmt->bytes_per_sec = cfg->bytes_per_sec; + fmt->blk_align = cfg->block_align; + fmt->bits_per_sample = cfg->bits_per_sample; + fmt->channel_mask = cfg->channel_mask; + fmt->enc_options = cfg->enc_options; + fmt->reserved = 0; + + rc = q6asm_ac_send_cmd_sync(ac, pkt); + kfree(pkt); + + return rc; +} +EXPORT_SYMBOL_GPL(q6asm_stream_media_format_block_wma_v9); + +int q6asm_stream_media_format_block_wma_v10(struct audio_client *ac, + struct q6asm_wma_cfg *cfg) +{ + struct asm_wmaprov10_fmt_blk_v2 *fmt; + struct apr_pkt *pkt; + void *p; + int rc, pkt_size; + + pkt_size = APR_HDR_SIZE + sizeof(*fmt); + p = kzalloc(pkt_size, GFP_KERNEL); + if (!p) + return -ENOMEM; + + pkt = p; + fmt = p + APR_HDR_SIZE; + + q6asm_add_hdr(ac, &pkt->hdr, pkt_size, true, ac->stream_id); + + pkt->hdr.opcode = ASM_DATA_CMD_MEDIA_FMT_UPDATE_V2; + fmt->fmt_blk.fmt_blk_size = sizeof(*fmt) - sizeof(fmt->fmt_blk); + fmt->fmtag = cfg->fmtag; + fmt->num_channels = cfg->num_channels; + fmt->sample_rate = cfg->sample_rate; + fmt->bytes_per_sec = cfg->bytes_per_sec; + fmt->blk_align = cfg->block_align; + fmt->bits_per_sample = cfg->bits_per_sample; + fmt->channel_mask = cfg->channel_mask; + fmt->enc_options = cfg->enc_options; + fmt->advanced_enc_options1 = cfg->adv_enc_options; + fmt->advanced_enc_options2 = cfg->adv_enc_options2; + + rc = q6asm_ac_send_cmd_sync(ac, pkt); + kfree(pkt); + + return rc; +} +EXPORT_SYMBOL_GPL(q6asm_stream_media_format_block_wma_v10); + +int q6asm_stream_media_format_block_alac(struct audio_client *ac, + struct q6asm_alac_cfg *cfg) +{ + struct asm_alac_fmt_blk_v2 *fmt; + struct apr_pkt *pkt; + void *p; + int rc, pkt_size; + + pkt_size = APR_HDR_SIZE + sizeof(*fmt); + p = kzalloc(pkt_size, GFP_KERNEL); + if (!p) + return -ENOMEM; + + pkt = p; + fmt = p + APR_HDR_SIZE; + + q6asm_add_hdr(ac, &pkt->hdr, pkt_size, true, ac->stream_id); + + pkt->hdr.opcode = ASM_DATA_CMD_MEDIA_FMT_UPDATE_V2; + fmt->fmt_blk.fmt_blk_size = sizeof(*fmt) - sizeof(fmt->fmt_blk); + + fmt->frame_length = cfg->frame_length; + fmt->compatible_version = cfg->compatible_version; + fmt->bit_depth = cfg->bit_depth; + fmt->num_channels = cfg->num_channels; + fmt->max_run = cfg->max_run; + fmt->max_frame_bytes = cfg->max_frame_bytes; + fmt->avg_bit_rate = cfg->avg_bit_rate; + fmt->sample_rate = cfg->sample_rate; + fmt->channel_layout_tag = cfg->channel_layout_tag; + fmt->pb = cfg->pb; + fmt->mb = cfg->mb; + fmt->kb = cfg->kb; + + rc = q6asm_ac_send_cmd_sync(ac, pkt); + kfree(pkt); + + return rc; +} +EXPORT_SYMBOL_GPL(q6asm_stream_media_format_block_alac); + +int q6asm_stream_media_format_block_ape(struct audio_client *ac, + struct q6asm_ape_cfg *cfg) +{ + struct asm_ape_fmt_blk_v2 *fmt; + struct apr_pkt *pkt; + void *p; + int rc, pkt_size; + + pkt_size = APR_HDR_SIZE + sizeof(*fmt); + p = kzalloc(pkt_size, GFP_KERNEL); + if (!p) + return -ENOMEM; + + pkt = p; + fmt = p + APR_HDR_SIZE; + + q6asm_add_hdr(ac, &pkt->hdr, pkt_size, true, ac->stream_id); + + pkt->hdr.opcode = ASM_DATA_CMD_MEDIA_FMT_UPDATE_V2; + fmt->fmt_blk.fmt_blk_size = sizeof(*fmt) - sizeof(fmt->fmt_blk); + + fmt->compatible_version = cfg->compatible_version; + fmt->compression_level = cfg->compression_level; + fmt->format_flags = cfg->format_flags; + fmt->blocks_per_frame = cfg->blocks_per_frame; + fmt->final_frame_blocks = cfg->final_frame_blocks; + fmt->total_frames = cfg->total_frames; + fmt->bits_per_sample = cfg->bits_per_sample; + fmt->num_channels = cfg->num_channels; + fmt->sample_rate = cfg->sample_rate; + fmt->seek_table_present = cfg->seek_table_present; + + rc = q6asm_ac_send_cmd_sync(ac, pkt); + kfree(pkt); + + return rc; +} +EXPORT_SYMBOL_GPL(q6asm_stream_media_format_block_ape); + /** * q6asm_enc_cfg_blk_pcm_format_support() - setup pcm configuration for capture * diff --git a/sound/soc/qcom/qdsp6/q6asm.h b/sound/soc/qcom/qdsp6/q6asm.h index 6764f55f7078..38a207d6cd95 100644 --- a/sound/soc/qcom/qdsp6/q6asm.h +++ b/sound/soc/qcom/qdsp6/q6asm.h @@ -45,6 +45,47 @@ struct q6asm_flac_cfg { u16 md5_sum; }; +struct q6asm_wma_cfg { + u32 fmtag; + u32 num_channels; + u32 sample_rate; + u32 bytes_per_sec; + u32 block_align; + u32 bits_per_sample; + u32 channel_mask; + u32 enc_options; + u32 adv_enc_options; + u32 adv_enc_options2; +}; + +struct q6asm_alac_cfg { + u32 frame_length; + u8 compatible_version; + u8 bit_depth; + u8 pb; + u8 mb; + u8 kb; + u8 num_channels; + u16 max_run; + u32 max_frame_bytes; + u32 avg_bit_rate; + u32 sample_rate; + u32 channel_layout_tag; +}; + +struct q6asm_ape_cfg { + u16 compatible_version; + u16 compression_level; + u32 format_flags; + u32 blocks_per_frame; + u32 final_frame_blocks; + u32 total_frames; + u16 bits_per_sample; + u16 num_channels; + u32 sample_rate; + u32 seek_table_present; +}; + typedef void (*q6asm_cb) (uint32_t opcode, uint32_t token, void *payload, void *priv); struct audio_client; @@ -55,7 +96,7 @@ void q6asm_audio_client_free(struct audio_client *ac); int q6asm_write_async(struct audio_client *ac, uint32_t len, uint32_t msw_ts, uint32_t lsw_ts, uint32_t flags); int q6asm_open_write(struct audio_client *ac, uint32_t format, - uint16_t bits_per_sample); + u32 codec_profile, uint16_t bits_per_sample); int q6asm_open_read(struct audio_client *ac, uint32_t format, uint16_t bits_per_sample); @@ -69,6 +110,14 @@ int q6asm_media_format_block_multi_ch_pcm(struct audio_client *ac, uint16_t bits_per_sample); int q6asm_stream_media_format_block_flac(struct audio_client *ac, struct q6asm_flac_cfg *cfg); +int q6asm_stream_media_format_block_wma_v9(struct audio_client *ac, + struct q6asm_wma_cfg *cfg); +int q6asm_stream_media_format_block_wma_v10(struct audio_client *ac, + struct q6asm_wma_cfg *cfg); +int q6asm_stream_media_format_block_alac(struct audio_client *ac, + struct q6asm_alac_cfg *cfg); +int q6asm_stream_media_format_block_ape(struct audio_client *ac, + struct q6asm_ape_cfg *cfg); int q6asm_run(struct audio_client *ac, uint32_t flags, uint32_t msw_ts, uint32_t lsw_ts); int q6asm_run_nowait(struct audio_client *ac, uint32_t flags, uint32_t msw_ts, diff --git a/sound/soc/qcom/qdsp6/q6routing.c b/sound/soc/qcom/qdsp6/q6routing.c index 20724102e85a..46e50612b92c 100644 --- a/sound/soc/qcom/qdsp6/q6routing.c +++ b/sound/soc/qcom/qdsp6/q6routing.c @@ -918,25 +918,6 @@ static const struct snd_soc_dapm_route intercon[] = { {"MM_UL6", NULL, "MultiMedia6 Mixer"}, {"MM_UL7", NULL, "MultiMedia7 Mixer"}, {"MM_UL8", NULL, "MultiMedia8 Mixer"}, - - {"MM_DL1", NULL, "MultiMedia1 Playback" }, - {"MM_DL2", NULL, "MultiMedia2 Playback" }, - {"MM_DL3", NULL, "MultiMedia3 Playback" }, - {"MM_DL4", NULL, "MultiMedia4 Playback" }, - {"MM_DL5", NULL, "MultiMedia5 Playback" }, - {"MM_DL6", NULL, "MultiMedia6 Playback" }, - {"MM_DL7", NULL, "MultiMedia7 Playback" }, - {"MM_DL8", NULL, "MultiMedia8 Playback" }, - - {"MultiMedia1 Capture", NULL, "MM_UL1"}, - {"MultiMedia2 Capture", NULL, "MM_UL2"}, - {"MultiMedia3 Capture", NULL, "MM_UL3"}, - {"MultiMedia4 Capture", NULL, "MM_UL4"}, - {"MultiMedia5 Capture", NULL, "MM_UL5"}, - {"MultiMedia6 Capture", NULL, "MM_UL6"}, - {"MultiMedia7 Capture", NULL, "MM_UL7"}, - {"MultiMedia8 Capture", NULL, "MM_UL8"}, - }; static int routing_hw_params(struct snd_soc_component *component, @@ -945,7 +926,7 @@ static int routing_hw_params(struct snd_soc_component *component, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct msm_routing_data *data = dev_get_drvdata(component->dev); - unsigned int be_id = rtd->cpu_dai->id; + unsigned int be_id = asoc_rtd_to_cpu(rtd, 0)->id; struct session_data *session; int path_type; diff --git a/sound/soc/qcom/sdm845.c b/sound/soc/qcom/sdm845.c index 3b5547a27aad..b2de65c7f95c 100644 --- a/sound/soc/qcom/sdm845.c +++ b/sound/soc/qcom/sdm845.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "common.h" #include "qdsp6/q6afe.h" @@ -31,10 +32,12 @@ struct sdm845_snd_data { struct snd_soc_jack jack; bool jack_setup; + bool stream_prepared[SLIM_MAX_RX_PORTS]; struct snd_soc_card *card; uint32_t pri_mi2s_clk_count; uint32_t sec_mi2s_clk_count; uint32_t quat_tdm_clk_count; + struct sdw_stream_runtime *sruntime[SLIM_MAX_RX_PORTS]; }; static unsigned int tdm_slot_offset[8] = {0, 4, 8, 12, 16, 20, 24, 28}; @@ -43,14 +46,21 @@ static int sdm845_slim_snd_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai_link *dai_link = rtd->dai_link; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai; + struct sdm845_snd_data *pdata = snd_soc_card_get_drvdata(rtd->card); u32 rx_ch[SLIM_MAX_RX_PORTS], tx_ch[SLIM_MAX_TX_PORTS]; + struct sdw_stream_runtime *sruntime; u32 rx_ch_cnt = 0, tx_ch_cnt = 0; int ret = 0, i; - for (i = 0 ; i < dai_link->num_codecs; i++) { - ret = snd_soc_dai_get_channel_map(rtd->codec_dais[i], + for_each_rtd_codec_dais(rtd, i, codec_dai) { + sruntime = snd_soc_dai_get_sdw_stream(codec_dai, + substream->stream); + if (sruntime != ERR_PTR(-ENOTSUPP)) + pdata->sruntime[cpu_dai->id] = sruntime; + + ret = snd_soc_dai_get_channel_map(codec_dai, &tx_ch_cnt, tx_ch, &rx_ch_cnt, rx_ch); if (ret != 0 && ret != -ENOTSUPP) { @@ -76,7 +86,8 @@ static int sdm845_tdm_snd_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai; int ret = 0, j; int channels, slot_width; @@ -125,8 +136,7 @@ static int sdm845_tdm_snd_hw_params(struct snd_pcm_substream *substream, } } - for (j = 0; j < rtd->num_codecs; j++) { - struct snd_soc_dai *codec_dai = rtd->codec_dais[j]; + for_each_rtd_codec_dais(rtd, j, codec_dai) { if (!strcmp(codec_dai->component->name_prefix, "Left")) { ret = snd_soc_dai_set_tdm_slot( @@ -161,8 +171,8 @@ static int sdm845_snd_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret = 0; switch (cpu_dai->id) { @@ -210,11 +220,10 @@ static int sdm845_dai_init(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_component *component; struct snd_soc_card *card = rtd->card; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct sdm845_snd_data *pdata = snd_soc_card_get_drvdata(card); struct snd_jack *jack; - struct snd_soc_dai_link *dai_link = rtd->dai_link; /* * Codec SLIMBUS configuration * RX1, RX2, RX3, RX4, RX5, RX6, RX7, RX8, RX9, RX10, RX11, RX12, RX13 @@ -266,8 +275,8 @@ static int sdm845_dai_init(struct snd_soc_pcm_runtime *rtd) } break; case SLIMBUS_0_RX...SLIMBUS_6_TX: - for (i = 0 ; i < dai_link->num_codecs; i++) { - rval = snd_soc_dai_set_channel_map(rtd->codec_dais[i], + for_each_rtd_codec_dais(rtd, i, codec_dai) { + rval = snd_soc_dai_set_channel_map(codec_dai, ARRAY_SIZE(tx_ch), tx_ch, ARRAY_SIZE(rx_ch), @@ -275,7 +284,7 @@ static int sdm845_dai_init(struct snd_soc_pcm_runtime *rtd) if (rval != 0 && rval != -ENOTSUPP) return rval; - snd_soc_dai_set_sysclk(rtd->codec_dais[i], 0, + snd_soc_dai_set_sysclk(codec_dai, 0, WCD934X_DEFAULT_MCLK_RATE, SNDRV_PCM_STREAM_PLAYBACK); } @@ -295,8 +304,8 @@ static int sdm845_snd_startup(struct snd_pcm_substream *substream) struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_card *card = rtd->card; struct sdm845_snd_data *data = snd_soc_card_get_drvdata(card); - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int j; int ret; @@ -345,8 +354,7 @@ static int sdm845_snd_startup(struct snd_pcm_substream *substream) codec_dai_fmt |= SND_SOC_DAIFMT_IB_NF | SND_SOC_DAIFMT_DSP_B; - for (j = 0; j < rtd->num_codecs; j++) { - codec_dai = rtd->codec_dais[j]; + for_each_rtd_codec_dais(rtd, j, codec_dai) { if (!strcmp(codec_dai->component->name_prefix, "Left")) { @@ -386,7 +394,7 @@ static void sdm845_snd_shutdown(struct snd_pcm_substream *substream) struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_card *card = rtd->card; struct sdm845_snd_data *data = snd_soc_card_get_drvdata(card); - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); switch (cpu_dai->id) { case PRIMARY_MI2S_RX: @@ -427,8 +435,65 @@ static void sdm845_snd_shutdown(struct snd_pcm_substream *substream) } } +static int sdm845_snd_prepare(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct sdm845_snd_data *data = snd_soc_card_get_drvdata(rtd->card); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id]; + int ret; + + if (!sruntime) + return 0; + + if (data->stream_prepared[cpu_dai->id]) { + sdw_disable_stream(sruntime); + sdw_deprepare_stream(sruntime); + data->stream_prepared[cpu_dai->id] = false; + } + + ret = sdw_prepare_stream(sruntime); + if (ret) + return ret; + + /** + * NOTE: there is a strict hw requirement about the ordering of port + * enables and actual WSA881x PA enable. PA enable should only happen + * after soundwire ports are enabled if not DC on the line is + * accumulated resulting in Click/Pop Noise + * PA enable/mute are handled as part of codec DAPM and digital mute. + */ + + ret = sdw_enable_stream(sruntime); + if (ret) { + sdw_deprepare_stream(sruntime); + return ret; + } + data->stream_prepared[cpu_dai->id] = true; + + return ret; +} + +static int sdm845_snd_hw_free(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct sdm845_snd_data *data = snd_soc_card_get_drvdata(rtd->card); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id]; + + if (sruntime && data->stream_prepared[cpu_dai->id]) { + sdw_disable_stream(sruntime); + sdw_deprepare_stream(sruntime); + data->stream_prepared[cpu_dai->id] = false; + } + + return 0; +} + static const struct snd_soc_ops sdm845_be_ops = { .hw_params = sdm845_snd_hw_params, + .hw_free = sdm845_snd_hw_free, + .prepare = sdm845_snd_prepare, .startup = sdm845_snd_startup, .shutdown = sdm845_snd_shutdown, }; diff --git a/sound/soc/qcom/storm.c b/sound/soc/qcom/storm.c index e6666e597265..3a6e18709b9e 100644 --- a/sound/soc/qcom/storm.c +++ b/sound/soc/qcom/storm.c @@ -39,7 +39,7 @@ static int storm_ops_hw_params(struct snd_pcm_substream *substream, */ sysclk_freq = rate * bitwidth * 2 * STORM_SYSCLK_MULT; - ret = snd_soc_dai_set_sysclk(soc_runtime->cpu_dai, 0, sysclk_freq, 0); + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(soc_runtime, 0), 0, sysclk_freq, 0); if (ret) { dev_err(card->dev, "error setting sysclk to %u: %d\n", sysclk_freq, ret); diff --git a/sound/soc/rockchip/rk3288_hdmi_analog.c b/sound/soc/rockchip/rk3288_hdmi_analog.c index 767700c34ee2..01078155a914 100644 --- a/sound/soc/rockchip/rk3288_hdmi_analog.c +++ b/sound/soc/rockchip/rk3288_hdmi_analog.c @@ -67,8 +67,8 @@ static int rk_hw_params(struct snd_pcm_substream *substream, { int ret = 0; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int mclk; switch (params_rate(params)) { diff --git a/sound/soc/rockchip/rk3399_gru_sound.c b/sound/soc/rockchip/rk3399_gru_sound.c index d951100bf770..f45e5aaa4b30 100644 --- a/sound/soc/rockchip/rk3399_gru_sound.c +++ b/sound/soc/rockchip/rk3399_gru_sound.c @@ -57,7 +57,7 @@ static int rockchip_sound_max98357a_hw_params(struct snd_pcm_substream *substrea mclk = params_rate(params) * SOUND_FS; - ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, 0, mclk, 0); + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(rtd, 0), 0, mclk, 0); if (ret) { dev_err(rtd->card->dev, "%s() error setting sysclk to %u: %d\n", __func__, mclk, ret); @@ -71,8 +71,8 @@ static int rockchip_sound_rt5514_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); unsigned int mclk; int ret; @@ -103,8 +103,8 @@ static int rockchip_sound_da7219_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int mclk, ret; /* in bypass mode, the mclk has to be one of the frequencies below */ @@ -153,8 +153,8 @@ static int rockchip_sound_da7219_hw_params(struct snd_pcm_substream *substream, static int rockchip_sound_da7219_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dais[0]->component; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; /* We need default MCLK and PLL settings for the accessory detection */ @@ -206,7 +206,7 @@ static int rockchip_sound_dmic_hw_params(struct snd_pcm_substream *substream, mclk = params_rate(params) * SOUND_FS; - ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, 0, mclk, 0); + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(rtd, 0), 0, mclk, 0); if (ret) { dev_err(rtd->card->dev, "%s() error setting sysclk to %u: %d\n", __func__, mclk, ret); diff --git a/sound/soc/rockchip/rockchip_max98090.c b/sound/soc/rockchip/rockchip_max98090.c index 60930fa85aa4..1f527d3763ce 100644 --- a/sound/soc/rockchip/rockchip_max98090.c +++ b/sound/soc/rockchip/rockchip_max98090.c @@ -146,8 +146,8 @@ static int rk_aif1_hw_params(struct snd_pcm_substream *substream, { int ret = 0; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int mclk; switch (params_rate(params)) { @@ -227,7 +227,7 @@ static struct snd_soc_jack rk_hdmi_jack; static int rk_hdmi_init(struct snd_soc_pcm_runtime *runtime) { struct snd_soc_card *card = runtime->card; - struct snd_soc_component *component = runtime->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(runtime, 0)->component; int ret; /* enable jack detection */ diff --git a/sound/soc/rockchip/rockchip_rt5645.c b/sound/soc/rockchip/rockchip_rt5645.c index 26b67b245484..0617ccf4e42c 100644 --- a/sound/soc/rockchip/rockchip_rt5645.c +++ b/sound/soc/rockchip/rockchip_rt5645.c @@ -56,8 +56,8 @@ static int rk_aif1_hw_params(struct snd_pcm_substream *substream, { int ret = 0; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int mclk; switch (params_rate(params)) { @@ -113,7 +113,7 @@ static int rk_init(struct snd_soc_pcm_runtime *runtime) return ret; } - return rt5645_set_jack_detect(runtime->codec_dai->component, + return rt5645_set_jack_detect(asoc_rtd_to_codec(runtime, 0)->component, &headset_jack, &headset_jack, &headset_jack); diff --git a/sound/soc/samsung/Kconfig b/sound/soc/samsung/Kconfig index 1a0b163ca47b..112911dc271b 100644 --- a/sound/soc/samsung/Kconfig +++ b/sound/soc/samsung/Kconfig @@ -151,7 +151,7 @@ config SND_SOC_TOBERMORY config SND_SOC_BELLS tristate "Audio support for Wolfson Bells" - depends on MFD_ARIZONA && I2C && SPI_MASTER + depends on MFD_ARIZONA && MFD_WM5102 && MFD_WM5110 && I2C && SPI_MASTER depends on MACH_WLF_CRAGG_6410 || COMPILE_TEST select SND_SAMSUNG_I2S select SND_SOC_WM5102 @@ -204,7 +204,7 @@ config SND_SOC_ARNDALE config SND_SOC_SAMSUNG_TM2_WM5110 tristate "SoC I2S Audio support for WM5110 on TM2 board" - depends on SND_SOC_SAMSUNG && MFD_ARIZONA && I2C && SPI_MASTER + depends on SND_SOC_SAMSUNG && MFD_ARIZONA && MFD_WM5110 && I2C && SPI_MASTER depends on GPIOLIB || COMPILE_TEST select SND_SOC_MAX98504 select SND_SOC_WM5110 diff --git a/sound/soc/samsung/arndale.c b/sound/soc/samsung/arndale.c index d64602950cbd..c81ece78e036 100644 --- a/sound/soc/samsung/arndale.c +++ b/sound/soc/samsung/arndale.c @@ -21,8 +21,8 @@ static int arndale_rt5631_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int rfs, ret; unsigned long rclk; @@ -56,7 +56,7 @@ static int arndale_wm1811_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); unsigned int rfs, rclk; /* Ensure AIF1CLK is >= 3 MHz for optimal performance */ @@ -174,7 +174,9 @@ static int arndale_audio_probe(struct platform_device *pdev) ret = devm_snd_soc_register_card(card->dev, card); if (ret) { - dev_err(&pdev->dev, "snd_soc_register_card() failed: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, + "snd_soc_register_card() failed: %d\n", ret); goto err_put_of_nodes; } return 0; diff --git a/sound/soc/samsung/bells.c b/sound/soc/samsung/bells.c index 5de633497f83..8b83f39c3ac9 100644 --- a/sound/soc/samsung/bells.c +++ b/sound/soc/samsung/bells.c @@ -60,7 +60,7 @@ static int bells_set_bias_level(struct snd_soc_card *card, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[DAI_DSP_CODEC]); - codec_dai = rtd->codec_dai; + codec_dai = asoc_rtd_to_codec(rtd, 0); component = codec_dai->component; if (dapm->dev != codec_dai->dev) @@ -106,7 +106,7 @@ static int bells_set_bias_level_post(struct snd_soc_card *card, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[DAI_DSP_CODEC]); - codec_dai = rtd->codec_dai; + codec_dai = asoc_rtd_to_codec(rtd, 0); component = codec_dai->component; if (dapm->dev != codec_dai->dev) @@ -152,11 +152,11 @@ static int bells_late_probe(struct snd_soc_card *card) int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[DAI_AP_DSP]); - wm0010 = rtd->codec_dai->component; + wm0010 = asoc_rtd_to_codec(rtd, 0)->component; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[DAI_DSP_CODEC]); - component = rtd->codec_dai->component; - aif1_dai = rtd->codec_dai; + component = asoc_rtd_to_codec(rtd, 0)->component; + aif1_dai = asoc_rtd_to_codec(rtd, 0); ret = snd_soc_component_set_sysclk(component, ARIZONA_CLK_SYSCLK, ARIZONA_CLK_SRC_FLL1, @@ -195,7 +195,7 @@ static int bells_late_probe(struct snd_soc_card *card) } rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[DAI_CODEC_CP]); - aif2_dai = rtd->cpu_dai; + aif2_dai = asoc_rtd_to_cpu(rtd, 0); ret = snd_soc_dai_set_sysclk(aif2_dai, ARIZONA_CLK_ASYNCCLK, 0, 0); if (ret != 0) { @@ -207,8 +207,8 @@ static int bells_late_probe(struct snd_soc_card *card) return 0; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[DAI_CODEC_SUB]); - aif3_dai = rtd->cpu_dai; - wm9081_dai = rtd->codec_dai; + aif3_dai = asoc_rtd_to_cpu(rtd, 0); + wm9081_dai = asoc_rtd_to_codec(rtd, 0); ret = snd_soc_dai_set_sysclk(aif3_dai, ARIZONA_CLK_SYSCLK, 0, 0); if (ret != 0) { diff --git a/sound/soc/samsung/h1940_uda1380.c b/sound/soc/samsung/h1940_uda1380.c index a95c34e53a2b..9139a1e7e200 100644 --- a/sound/soc/samsung/h1940_uda1380.c +++ b/sound/soc/samsung/h1940_uda1380.c @@ -68,7 +68,7 @@ static int h1940_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int div; int ret; unsigned int rate = params_rate(params); diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c index a57bb989a0ef..f86e3028b402 100644 --- a/sound/soc/samsung/i2s.c +++ b/sound/soc/samsung/i2s.c @@ -932,7 +932,7 @@ static int i2s_trigger(struct snd_pcm_substream *substream, struct samsung_i2s_priv *priv = snd_soc_dai_get_drvdata(dai); int capture = (substream->stream == SNDRV_PCM_STREAM_CAPTURE); struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct i2s_dai *i2s = to_info(rtd->cpu_dai); + struct i2s_dai *i2s = to_info(asoc_rtd_to_cpu(rtd, 0)); unsigned long flags; switch (cmd) { diff --git a/sound/soc/samsung/jive_wm8750.c b/sound/soc/samsung/jive_wm8750.c index 949d2e029962..30899016cf08 100644 --- a/sound/soc/samsung/jive_wm8750.c +++ b/sound/soc/samsung/jive_wm8750.c @@ -33,8 +33,8 @@ static int jive_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct s3c_i2sv2_rate_calc div; unsigned int clk = 0; int ret = 0; diff --git a/sound/soc/samsung/littlemill.c b/sound/soc/samsung/littlemill.c index 59904f44118b..f4375c49f7f4 100644 --- a/sound/soc/samsung/littlemill.c +++ b/sound/soc/samsung/littlemill.c @@ -23,7 +23,7 @@ static int littlemill_set_bias_level(struct snd_soc_card *card, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - aif1_dai = rtd->codec_dai; + aif1_dai = asoc_rtd_to_codec(rtd, 0); if (dapm->dev != aif1_dai->dev) return 0; @@ -70,7 +70,7 @@ static int littlemill_set_bias_level_post(struct snd_soc_card *card, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - aif1_dai = rtd->codec_dai; + aif1_dai = asoc_rtd_to_codec(rtd, 0); if (dapm->dev != aif1_dai->dev) return 0; @@ -105,7 +105,7 @@ static int littlemill_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; sample_rate = params_rate(params); @@ -181,7 +181,7 @@ static int bbclk_ev(struct snd_soc_dapm_widget *w, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[1]); - aif2_dai = rtd->cpu_dai; + aif2_dai = asoc_rtd_to_cpu(rtd, 0); switch (event) { case SND_SOC_DAPM_PRE_PMU: @@ -264,11 +264,11 @@ static int littlemill_late_probe(struct snd_soc_card *card) int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - component = rtd->codec_dai->component; - aif1_dai = rtd->codec_dai; + component = asoc_rtd_to_codec(rtd, 0)->component; + aif1_dai = asoc_rtd_to_codec(rtd, 0); rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[1]); - aif2_dai = rtd->cpu_dai; + aif2_dai = asoc_rtd_to_cpu(rtd, 0); ret = snd_soc_dai_set_sysclk(aif1_dai, WM8994_SYSCLK_MCLK2, 32768, SND_SOC_CLOCK_IN); @@ -325,7 +325,7 @@ static int littlemill_probe(struct platform_device *pdev) card->dev = &pdev->dev; ret = devm_snd_soc_register_card(&pdev->dev, card); - if (ret) + if (ret && ret != -EPROBE_DEFER) dev_err(&pdev->dev, "snd_soc_register_card() failed: %d\n", ret); diff --git a/sound/soc/samsung/lowland.c b/sound/soc/samsung/lowland.c index 098eefc764db..998d10cf8c94 100644 --- a/sound/soc/samsung/lowland.c +++ b/sound/soc/samsung/lowland.c @@ -32,7 +32,7 @@ static struct snd_soc_jack_pin lowland_headset_pins[] = { static int lowland_wm5100_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; int ret; ret = snd_soc_component_set_sysclk(component, WM5100_CLK_SYSCLK, @@ -65,7 +65,7 @@ static int lowland_wm5100_init(struct snd_soc_pcm_runtime *rtd) static int lowland_wm9081_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; snd_soc_dapm_nc_pin(&rtd->card->dapm, "LINEOUT"); @@ -183,7 +183,7 @@ static int lowland_probe(struct platform_device *pdev) card->dev = &pdev->dev; ret = devm_snd_soc_register_card(&pdev->dev, card); - if (ret) + if (ret && ret != -EPROBE_DEFER) dev_err(&pdev->dev, "snd_soc_register_card() failed: %d\n", ret); diff --git a/sound/soc/samsung/neo1973_wm8753.c b/sound/soc/samsung/neo1973_wm8753.c index 1339e41e9860..b7ce1da854ce 100644 --- a/sound/soc/samsung/neo1973_wm8753.c +++ b/sound/soc/samsung/neo1973_wm8753.c @@ -26,8 +26,8 @@ static int neo1973_hifi_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int pll_out = 0, bclk = 0; int ret = 0; unsigned long iis_clkrate; @@ -100,7 +100,7 @@ static int neo1973_hifi_hw_params(struct snd_pcm_substream *substream, static int neo1973_hifi_hw_free(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); /* disable the PLL */ return snd_soc_dai_set_pll(codec_dai, WM8753_PLL1, 0, 0, 0); @@ -118,7 +118,7 @@ static int neo1973_voice_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); unsigned int pcmdiv = 0; int ret = 0; unsigned long iis_clkrate; @@ -155,7 +155,7 @@ static int neo1973_voice_hw_params(struct snd_pcm_substream *substream, static int neo1973_voice_hw_free(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); /* disable the PLL */ return snd_soc_dai_set_pll(codec_dai, WM8753_PLL2, 0, 0, 0); diff --git a/sound/soc/samsung/odroid.c b/sound/soc/samsung/odroid.c index f0f5fa9c27d3..6eda5af989fe 100644 --- a/sound/soc/samsung/odroid.c +++ b/sound/soc/samsung/odroid.c @@ -98,7 +98,7 @@ static int odroid_card_be_hw_params(struct snd_pcm_substream *substream, return ret; if (rtd->num_codecs > 1) { - struct snd_soc_dai *codec_dai = rtd->codec_dais[1]; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 1); ret = snd_soc_dai_set_sysclk(codec_dai, 0, rclk_freq, SND_SOC_CLOCK_IN); @@ -311,7 +311,9 @@ static int odroid_audio_probe(struct platform_device *pdev) ret = devm_snd_soc_register_card(dev, card); if (ret < 0) { - dev_err(dev, "snd_soc_register_card() failed: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "snd_soc_register_card() failed: %d\n", + ret); goto err_put_clk_i2s; } diff --git a/sound/soc/samsung/pcm.c b/sound/soc/samsung/pcm.c index f6e67d0e7882..a5b1a12b3496 100644 --- a/sound/soc/samsung/pcm.c +++ b/sound/soc/samsung/pcm.c @@ -212,7 +212,7 @@ static int s3c_pcm_trigger(struct snd_pcm_substream *substream, int cmd, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct s3c_pcm_info *pcm = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct s3c_pcm_info *pcm = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); unsigned long flags; dev_dbg(pcm->dev, "Entered %s\n", __func__); @@ -256,7 +256,7 @@ static int s3c_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *socdai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct s3c_pcm_info *pcm = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct s3c_pcm_info *pcm = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); void __iomem *regs = pcm->regs; struct clk *clk; int sclk_div, sync_div; diff --git a/sound/soc/samsung/rx1950_uda1380.c b/sound/soc/samsung/rx1950_uda1380.c index 4b247e91ae5b..3afe63c0923e 100644 --- a/sound/soc/samsung/rx1950_uda1380.c +++ b/sound/soc/samsung/rx1950_uda1380.c @@ -149,7 +149,7 @@ static int rx1950_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int div; int ret; unsigned int rate = params_rate(params); diff --git a/sound/soc/samsung/s3c-i2s-v2.c b/sound/soc/samsung/s3c-i2s-v2.c index 593be1b668d6..358887848293 100644 --- a/sound/soc/samsung/s3c-i2s-v2.c +++ b/sound/soc/samsung/s3c-i2s-v2.c @@ -380,7 +380,7 @@ static int s3c2412_i2s_trigger(struct snd_pcm_substream *substream, int cmd, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct s3c_i2sv2_info *i2s = to_info(rtd->cpu_dai); + struct s3c_i2sv2_info *i2s = to_info(asoc_rtd_to_cpu(rtd, 0)); int capture = (substream->stream == SNDRV_PCM_STREAM_CAPTURE); unsigned long irqs; int ret = 0; diff --git a/sound/soc/samsung/s3c24xx_simtec.c b/sound/soc/samsung/s3c24xx_simtec.c index 4543705b8d87..fd2a4da086f3 100644 --- a/sound/soc/samsung/s3c24xx_simtec.c +++ b/sound/soc/samsung/s3c24xx_simtec.c @@ -160,8 +160,8 @@ static int simtec_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(codec_dai, 0, diff --git a/sound/soc/samsung/s3c24xx_uda134x.c b/sound/soc/samsung/s3c24xx_uda134x.c index 55d2a802a6cb..abb5c4713c53 100644 --- a/sound/soc/samsung/s3c24xx_uda134x.c +++ b/sound/soc/samsung/s3c24xx_uda134x.c @@ -51,7 +51,7 @@ static int s3c24xx_uda134x_startup(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct s3c24xx_uda134x *priv = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int ret = 0; mutex_lock(&priv->clk_lock); @@ -119,8 +119,8 @@ static int s3c24xx_uda134x_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int clk = 0; int ret = 0; int clk_source, fs_mode; diff --git a/sound/soc/samsung/smartq_wm8987.c b/sound/soc/samsung/smartq_wm8987.c index fab3db9fdb98..36bef136d57f 100644 --- a/sound/soc/samsung/smartq_wm8987.c +++ b/sound/soc/samsung/smartq_wm8987.c @@ -25,8 +25,8 @@ static int smartq_hifi_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int clk = 0; int ret; diff --git a/sound/soc/samsung/smdk_spdif.c b/sound/soc/samsung/smdk_spdif.c index 4baef84d29ee..776a270261bf 100644 --- a/sound/soc/samsung/smdk_spdif.c +++ b/sound/soc/samsung/smdk_spdif.c @@ -101,7 +101,7 @@ static int smdk_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned long pll_out, rclk_rate; int ret, ratio; diff --git a/sound/soc/samsung/smdk_wm8580.c b/sound/soc/samsung/smdk_wm8580.c index d096ff912260..02074c34a2b2 100644 --- a/sound/soc/samsung/smdk_wm8580.c +++ b/sound/soc/samsung/smdk_wm8580.c @@ -23,7 +23,7 @@ static int smdk_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); unsigned int pll_out; int rfs, ret; diff --git a/sound/soc/samsung/smdk_wm8994.c b/sound/soc/samsung/smdk_wm8994.c index 28f8be000aa1..a9f345f19a8a 100644 --- a/sound/soc/samsung/smdk_wm8994.c +++ b/sound/soc/samsung/smdk_wm8994.c @@ -45,7 +45,7 @@ static int smdk_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); unsigned int pll_out; int ret; @@ -178,7 +178,7 @@ static int smdk_audio_probe(struct platform_device *pdev) ret = devm_snd_soc_register_card(&pdev->dev, card); - if (ret) + if (ret && ret != -EPROBE_DEFER) dev_err(&pdev->dev, "snd_soc_register_card() failed:%d\n", ret); return ret; diff --git a/sound/soc/samsung/smdk_wm8994pcm.c b/sound/soc/samsung/smdk_wm8994pcm.c index 2e3dc7320c62..746930dde5d7 100644 --- a/sound/soc/samsung/smdk_wm8994pcm.c +++ b/sound/soc/samsung/smdk_wm8994pcm.c @@ -44,8 +44,8 @@ static int smdk_wm8994_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned long mclk_freq; int rfs, ret; @@ -118,7 +118,7 @@ static int snd_smdk_probe(struct platform_device *pdev) smdk_pcm.dev = &pdev->dev; ret = devm_snd_soc_register_card(&pdev->dev, &smdk_pcm); - if (ret) + if (ret && ret != -EPROBE_DEFER) dev_err(&pdev->dev, "snd_soc_register_card failed %d\n", ret); return ret; diff --git a/sound/soc/samsung/snow.c b/sound/soc/samsung/snow.c index f075aae9561a..40c5de8df0ff 100644 --- a/sound/soc/samsung/snow.c +++ b/sound/soc/samsung/snow.c @@ -110,9 +110,9 @@ static int snow_late_probe(struct snd_soc_card *card) /* In the multi-codec case codec_dais 0 is MAX98095 and 1 is HDMI. */ if (rtd->num_codecs > 1) - codec_dai = rtd->codec_dais[0]; + codec_dai = asoc_rtd_to_codec(rtd, 0); else - codec_dai = rtd->codec_dai; + codec_dai = asoc_rtd_to_codec(rtd, 0); /* Set the MCLK rate for the codec */ return snd_soc_dai_set_sysclk(codec_dai, 0, @@ -216,7 +216,9 @@ static int snow_probe(struct platform_device *pdev) ret = devm_snd_soc_register_card(dev, card); if (ret) { - dev_err(&pdev->dev, "snd_soc_register_card failed (%d)\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, + "snd_soc_register_card failed (%d)\n", ret); return ret; } diff --git a/sound/soc/samsung/spdif.c b/sound/soc/samsung/spdif.c index 1a9f08a50394..759fc6644329 100644 --- a/sound/soc/samsung/spdif.c +++ b/sound/soc/samsung/spdif.c @@ -142,7 +142,7 @@ static int spdif_trigger(struct snd_pcm_substream *substream, int cmd, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct samsung_spdif_info *spdif = to_info(rtd->cpu_dai); + struct samsung_spdif_info *spdif = to_info(asoc_rtd_to_cpu(rtd, 0)); unsigned long flags; dev_dbg(spdif->dev, "Entered %s\n", __func__); @@ -178,7 +178,7 @@ static int spdif_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *socdai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct samsung_spdif_info *spdif = to_info(rtd->cpu_dai); + struct samsung_spdif_info *spdif = to_info(asoc_rtd_to_cpu(rtd, 0)); void __iomem *regs = spdif->regs; struct snd_dmaengine_dai_dma_data *dma_data; u32 con, clkcon, cstas; @@ -194,7 +194,7 @@ static int spdif_hw_params(struct snd_pcm_substream *substream, return -EINVAL; } - snd_soc_dai_set_dma_data(rtd->cpu_dai, substream, dma_data); + snd_soc_dai_set_dma_data(asoc_rtd_to_cpu(rtd, 0), substream, dma_data); spin_lock_irqsave(&spdif->lock, flags); @@ -280,7 +280,7 @@ static void spdif_shutdown(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct samsung_spdif_info *spdif = to_info(rtd->cpu_dai); + struct samsung_spdif_info *spdif = to_info(asoc_rtd_to_cpu(rtd, 0)); void __iomem *regs = spdif->regs; u32 con, clkcon; diff --git a/sound/soc/samsung/speyside.c b/sound/soc/samsung/speyside.c index ea0d1ec67f01..f5f6ba00d073 100644 --- a/sound/soc/samsung/speyside.c +++ b/sound/soc/samsung/speyside.c @@ -25,7 +25,7 @@ static int speyside_set_bias_level(struct snd_soc_card *card, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[1]); - codec_dai = rtd->codec_dai; + codec_dai = asoc_rtd_to_codec(rtd, 0); if (dapm->dev != codec_dai->dev) return 0; @@ -61,7 +61,7 @@ static int speyside_set_bias_level_post(struct snd_soc_card *card, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[1]); - codec_dai = rtd->codec_dai; + codec_dai = asoc_rtd_to_codec(rtd, 0); if (dapm->dev != codec_dai->dev) return 0; @@ -131,7 +131,7 @@ static void speyside_set_polarity(struct snd_soc_component *component, static int speyside_wm0010_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); int ret; ret = snd_soc_dai_set_sysclk(dai, 0, MCLK_AUDIO_RATE, 0); @@ -143,7 +143,7 @@ static int speyside_wm0010_init(struct snd_soc_pcm_runtime *rtd) static int speyside_wm8996_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *dai = rtd->codec_dai; + struct snd_soc_dai *dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_component *component = dai->component; int ret; @@ -330,7 +330,7 @@ static int speyside_probe(struct platform_device *pdev) card->dev = &pdev->dev; ret = devm_snd_soc_register_card(&pdev->dev, card); - if (ret) + if (ret && ret != -EPROBE_DEFER) dev_err(&pdev->dev, "snd_soc_register_card() failed: %d\n", ret); diff --git a/sound/soc/samsung/tm2_wm5110.c b/sound/soc/samsung/tm2_wm5110.c index 10ff14b856f2..6dfd540e2d74 100644 --- a/sound/soc/samsung/tm2_wm5110.c +++ b/sound/soc/samsung/tm2_wm5110.c @@ -93,7 +93,7 @@ static int tm2_aif1_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct tm2_machine_priv *priv = snd_soc_card_get_drvdata(rtd->card); switch (params_rate(params)) { @@ -134,7 +134,7 @@ static int tm2_aif2_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; unsigned int asyncclk_rate; int ret; @@ -188,7 +188,7 @@ static int tm2_aif2_hw_params(struct snd_pcm_substream *substream, static int tm2_aif2_hw_free(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; int ret; /* disable FLL2 */ @@ -209,7 +209,7 @@ static int tm2_hdmi_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); unsigned int bfs; int bitwidth, ret; @@ -284,7 +284,7 @@ static int tm2_set_bias_level(struct snd_soc_card *card, rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - if (dapm->dev != rtd->codec_dai->dev) + if (dapm->dev != asoc_rtd_to_codec(rtd, 0)->dev) return 0; switch (level) { @@ -315,8 +315,8 @@ static int tm2_late_probe(struct snd_soc_card *card) int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[TM2_DAI_AIF1]); - aif1_dai = rtd->codec_dai; - priv->component = rtd->codec_dai->component; + aif1_dai = asoc_rtd_to_codec(rtd, 0); + priv->component = asoc_rtd_to_codec(rtd, 0)->component; ret = snd_soc_dai_set_sysclk(aif1_dai, ARIZONA_CLK_SYSCLK, 0, 0); if (ret < 0) { @@ -325,7 +325,7 @@ static int tm2_late_probe(struct snd_soc_card *card) } rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[TM2_DAI_AIF2]); - aif2_dai = rtd->codec_dai; + aif2_dai = asoc_rtd_to_codec(rtd, 0); ret = snd_soc_dai_set_sysclk(aif2_dai, ARIZONA_CLK_ASYNCCLK, 0, 0); if (ret < 0) { @@ -611,7 +611,8 @@ static int tm2_probe(struct platform_device *pdev) ret = devm_snd_soc_register_card(dev, card); if (ret < 0) { - dev_err(dev, "Failed to register card: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to register card: %d\n", ret); goto dai_node_put; } diff --git a/sound/soc/samsung/tobermory.c b/sound/soc/samsung/tobermory.c index fdce28cc26c4..c962d2c2a7f7 100644 --- a/sound/soc/samsung/tobermory.c +++ b/sound/soc/samsung/tobermory.c @@ -23,7 +23,7 @@ static int tobermory_set_bias_level(struct snd_soc_card *card, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - codec_dai = rtd->codec_dai; + codec_dai = asoc_rtd_to_codec(rtd, 0); if (dapm->dev != codec_dai->dev) return 0; @@ -66,7 +66,7 @@ static int tobermory_set_bias_level_post(struct snd_soc_card *card, int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - codec_dai = rtd->codec_dai; + codec_dai = asoc_rtd_to_codec(rtd, 0); if (dapm->dev != codec_dai->dev) return 0; @@ -181,8 +181,8 @@ static int tobermory_late_probe(struct snd_soc_card *card) int ret; rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - component = rtd->codec_dai->component; - codec_dai = rtd->codec_dai; + component = asoc_rtd_to_codec(rtd, 0)->component; + codec_dai = asoc_rtd_to_codec(rtd, 0); ret = snd_soc_dai_set_sysclk(codec_dai, WM8962_SYSCLK_MCLK, 32768, SND_SOC_CLOCK_IN); @@ -229,7 +229,7 @@ static int tobermory_probe(struct platform_device *pdev) card->dev = &pdev->dev; ret = devm_snd_soc_register_card(&pdev->dev, card); - if (ret) + if (ret && ret != -EPROBE_DEFER) dev_err(&pdev->dev, "snd_soc_register_card() failed: %d\n", ret); diff --git a/sound/soc/sh/dma-sh7760.c b/sound/soc/sh/dma-sh7760.c index eee1a1e994cb..a35de78f14a9 100644 --- a/sound/soc/sh/dma-sh7760.c +++ b/sound/soc/sh/dma-sh7760.c @@ -119,7 +119,7 @@ static int camelot_pcm_open(struct snd_soc_component *component, struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct camelot_pcm *cam = &cam_pcm_data[rtd->cpu_dai->id]; + struct camelot_pcm *cam = &cam_pcm_data[asoc_rtd_to_cpu(rtd, 0)->id]; int recv = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? 0:1; int ret, dmairq; @@ -132,7 +132,7 @@ static int camelot_pcm_open(struct snd_soc_component *component, ret = dmabrg_request_irq(dmairq, camelot_rxdma, cam); if (unlikely(ret)) { pr_debug("audio unit %d irqs already taken!\n", - rtd->cpu_dai->id); + asoc_rtd_to_cpu(rtd, 0)->id); return -EBUSY; } (void)dmabrg_request_irq(dmairq + 1,camelot_rxdma, cam); @@ -141,7 +141,7 @@ static int camelot_pcm_open(struct snd_soc_component *component, ret = dmabrg_request_irq(dmairq, camelot_txdma, cam); if (unlikely(ret)) { pr_debug("audio unit %d irqs already taken!\n", - rtd->cpu_dai->id); + asoc_rtd_to_cpu(rtd, 0)->id); return -EBUSY; } (void)dmabrg_request_irq(dmairq + 1, camelot_txdma, cam); @@ -153,7 +153,7 @@ static int camelot_pcm_close(struct snd_soc_component *component, struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct camelot_pcm *cam = &cam_pcm_data[rtd->cpu_dai->id]; + struct camelot_pcm *cam = &cam_pcm_data[asoc_rtd_to_cpu(rtd, 0)->id]; int recv = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? 0:1; int dmairq; @@ -175,7 +175,7 @@ static int camelot_hw_params(struct snd_soc_component *component, struct snd_pcm_hw_params *hw_params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct camelot_pcm *cam = &cam_pcm_data[rtd->cpu_dai->id]; + struct camelot_pcm *cam = &cam_pcm_data[asoc_rtd_to_cpu(rtd, 0)->id]; int recv = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? 0:1; int ret; @@ -194,7 +194,7 @@ static int camelot_prepare(struct snd_soc_component *component, { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct camelot_pcm *cam = &cam_pcm_data[rtd->cpu_dai->id]; + struct camelot_pcm *cam = &cam_pcm_data[asoc_rtd_to_cpu(rtd, 0)->id]; pr_debug("PCM data: addr 0x%08lx len %d\n", (u32)runtime->dma_addr, runtime->dma_bytes); @@ -242,7 +242,7 @@ static int camelot_trigger(struct snd_soc_component *component, struct snd_pcm_substream *substream, int cmd) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct camelot_pcm *cam = &cam_pcm_data[rtd->cpu_dai->id]; + struct camelot_pcm *cam = &cam_pcm_data[asoc_rtd_to_cpu(rtd, 0)->id]; int recv = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? 0:1; switch (cmd) { @@ -270,7 +270,7 @@ static snd_pcm_uframes_t camelot_pos(struct snd_soc_component *component, { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct camelot_pcm *cam = &cam_pcm_data[rtd->cpu_dai->id]; + struct camelot_pcm *cam = &cam_pcm_data[asoc_rtd_to_cpu(rtd, 0)->id]; int recv = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? 0:1; unsigned long pos; diff --git a/sound/soc/sh/fsi.c b/sound/soc/sh/fsi.c index 4b35ef402604..1c3c4fdc9bef 100644 --- a/sound/soc/sh/fsi.c +++ b/sound/soc/sh/fsi.c @@ -408,7 +408,7 @@ static struct snd_soc_dai *fsi_get_dai(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - return rtd->cpu_dai; + return asoc_rtd_to_cpu(rtd, 0); } static struct fsi_priv *fsi_get_priv_frm_dai(struct snd_soc_dai *dai) @@ -1938,8 +1938,7 @@ static int fsi_probe(struct platform_device *pdev) if (!master) return -ENOMEM; - master->base = devm_ioremap(&pdev->dev, - res->start, resource_size(res)); + master->base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!master->base) { dev_err(&pdev->dev, "Unable to ioremap FSI registers.\n"); return -ENXIO; diff --git a/sound/soc/sh/migor.c b/sound/soc/sh/migor.c index 991557e25eba..d5702fbf176b 100644 --- a/sound/soc/sh/migor.c +++ b/sound/soc/sh/migor.c @@ -46,7 +46,7 @@ static int migor_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int ret; unsigned int rate = params_rate(params); @@ -67,7 +67,7 @@ static int migor_hw_params(struct snd_pcm_substream *substream, clk_set_rate(&siumckb_clk, codec_freq); dev_dbg(codec_dai->dev, "%s: configure %luHz\n", __func__, codec_freq); - ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, SIU_CLKB_EXT, + ret = snd_soc_dai_set_sysclk(asoc_rtd_to_cpu(rtd, 0), SIU_CLKB_EXT, codec_freq / 2, SND_SOC_CLOCK_IN); if (!ret) @@ -79,7 +79,7 @@ static int migor_hw_params(struct snd_pcm_substream *substream, static int migor_hw_free(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); if (use_count) { use_count--; diff --git a/sound/soc/sh/rcar/core.c b/sound/soc/sh/rcar/core.c index 0bfcb77e5f65..4349f2fb823f 100644 --- a/sound/soc/sh/rcar/core.c +++ b/sound/soc/sh/rcar/core.c @@ -696,7 +696,7 @@ struct snd_soc_dai *rsnd_substream_to_dai(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - return rtd->cpu_dai; + return asoc_rtd_to_cpu(rtd, 0); } static diff --git a/sound/soc/soc-compress.c b/sound/soc/soc-compress.c index 392a1c5b15d3..50062eb79adb 100644 --- a/sound/soc/soc-compress.c +++ b/sound/soc/soc-compress.c @@ -810,9 +810,10 @@ int snd_soc_new_compress(struct snd_soc_pcm_runtime *rtd, int num) int playback = 0, capture = 0; int i; - if (rtd->num_codecs > 1) { + if (rtd->num_cpus > 1 || + rtd->num_codecs > 1) { dev_err(rtd->card->dev, - "Compress ASoC: Multicodec not supported\n"); + "Compress ASoC: Multi CPU/Codec not supported\n"); return -EINVAL; } diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index d0c35ff2aae0..0f9aad6b8b73 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -365,19 +365,20 @@ EXPORT_SYMBOL_GPL(snd_soc_get_pcm_runtime); void snd_soc_close_delayed_work(struct snd_soc_pcm_runtime *rtd) { struct snd_soc_dai *codec_dai = rtd->codec_dai; + int playback = SNDRV_PCM_STREAM_PLAYBACK; mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass); dev_dbg(rtd->dev, "ASoC: pop wq checking: %s status: %s waiting: %s\n", codec_dai->driver->playback.stream_name, - codec_dai->playback_active ? "active" : "inactive", + codec_dai->stream_active[playback] ? "active" : "inactive", rtd->pop_wait ? "yes" : "no"); /* are we waiting on this codec DAI stream */ if (rtd->pop_wait == 1) { rtd->pop_wait = 0; - snd_soc_dapm_stream_event(rtd, SNDRV_PCM_STREAM_PLAYBACK, + snd_soc_dapm_stream_event(rtd, playback, SND_SOC_DAPM_STREAM_STOP); } @@ -431,6 +432,7 @@ static struct snd_soc_pcm_runtime *soc_new_pcm_runtime( struct snd_soc_component *component; struct device *dev; int ret; + int stream; /* * for rtd->dev @@ -465,22 +467,30 @@ static struct snd_soc_pcm_runtime *soc_new_pcm_runtime( rtd->dev = dev; INIT_LIST_HEAD(&rtd->list); - INIT_LIST_HEAD(&rtd->dpcm[SNDRV_PCM_STREAM_PLAYBACK].be_clients); - INIT_LIST_HEAD(&rtd->dpcm[SNDRV_PCM_STREAM_CAPTURE].be_clients); - INIT_LIST_HEAD(&rtd->dpcm[SNDRV_PCM_STREAM_PLAYBACK].fe_clients); - INIT_LIST_HEAD(&rtd->dpcm[SNDRV_PCM_STREAM_CAPTURE].fe_clients); + for_each_pcm_streams(stream) { + INIT_LIST_HEAD(&rtd->dpcm[stream].be_clients); + INIT_LIST_HEAD(&rtd->dpcm[stream].fe_clients); + } dev_set_drvdata(dev, rtd); INIT_DELAYED_WORK(&rtd->delayed_work, close_delayed_work); /* - * for rtd->codec_dais + * for rtd->dais */ - rtd->codec_dais = devm_kcalloc(dev, dai_link->num_codecs, + rtd->dais = devm_kcalloc(dev, dai_link->num_cpus + dai_link->num_codecs, sizeof(struct snd_soc_dai *), GFP_KERNEL); - if (!rtd->codec_dais) + if (!rtd->dais) goto free_rtd; + /* + * dais = [][][][][][][][][][][][][][][][][][] + * ^cpu_dais ^codec_dais + * |--- num_cpus ---|--- num_codecs --| + */ + rtd->cpu_dais = &rtd->dais[0]; + rtd->codec_dais = &rtd->dais[dai_link->num_cpus]; + /* * rtd remaining settings */ @@ -514,6 +524,7 @@ int snd_soc_suspend(struct device *dev) struct snd_soc_card *card = dev_get_drvdata(dev); struct snd_soc_component *component; struct snd_soc_pcm_runtime *rtd; + int playback = SNDRV_PCM_STREAM_PLAYBACK; int i; /* If the card is not initialized yet there is nothing to do */ @@ -536,10 +547,9 @@ int snd_soc_suspend(struct device *dev) if (rtd->dai_link->ignore_suspend) continue; - for_each_rtd_codec_dai(rtd, i, dai) { - if (dai->playback_active) - snd_soc_dai_digital_mute(dai, 1, - SNDRV_PCM_STREAM_PLAYBACK); + for_each_rtd_codec_dais(rtd, i, dai) { + if (dai->stream_active[playback]) + snd_soc_dai_digital_mute(dai, 1, playback); } } @@ -558,17 +568,14 @@ int snd_soc_suspend(struct device *dev) snd_soc_flush_all_delayed_work(card); for_each_card_rtds(card, rtd) { + int stream; if (rtd->dai_link->ignore_suspend) continue; - snd_soc_dapm_stream_event(rtd, - SNDRV_PCM_STREAM_PLAYBACK, - SND_SOC_DAPM_STREAM_SUSPEND); - - snd_soc_dapm_stream_event(rtd, - SNDRV_PCM_STREAM_CAPTURE, - SND_SOC_DAPM_STREAM_SUSPEND); + for_each_pcm_streams(stream) + snd_soc_dapm_stream_event(rtd, stream, + SND_SOC_DAPM_STREAM_SUSPEND); } /* Recheck all endpoints too, their state is affected by suspend */ @@ -664,30 +671,27 @@ static void soc_resume_deferred(struct work_struct *work) } for_each_card_rtds(card, rtd) { + int stream; if (rtd->dai_link->ignore_suspend) continue; - snd_soc_dapm_stream_event(rtd, - SNDRV_PCM_STREAM_PLAYBACK, - SND_SOC_DAPM_STREAM_RESUME); - - snd_soc_dapm_stream_event(rtd, - SNDRV_PCM_STREAM_CAPTURE, - SND_SOC_DAPM_STREAM_RESUME); + for_each_pcm_streams(stream) + snd_soc_dapm_stream_event(rtd, stream, + SND_SOC_DAPM_STREAM_RESUME); } /* unmute any active DACs */ for_each_card_rtds(card, rtd) { struct snd_soc_dai *dai; + int playback = SNDRV_PCM_STREAM_PLAYBACK; if (rtd->dai_link->ignore_suspend) continue; - for_each_rtd_codec_dai(rtd, i, dai) { - if (dai->playback_active) - snd_soc_dai_digital_mute(dai, 0, - SNDRV_PCM_STREAM_PLAYBACK); + for_each_rtd_codec_dais(rtd, i, dai) { + if (dai->stream_active[playback]) + snd_soc_dai_digital_mute(dai, 0, playback); } } @@ -837,7 +841,7 @@ static int soc_dai_link_sanity_check(struct snd_soc_card *card, struct snd_soc_dai_link *link) { int i; - struct snd_soc_dai_link_component *codec, *platform; + struct snd_soc_dai_link_component *cpu, *codec, *platform; for_each_link_codecs(link, i, codec) { /* @@ -886,44 +890,38 @@ static int soc_dai_link_sanity_check(struct snd_soc_card *card, return -EPROBE_DEFER; } - /* FIXME */ - if (link->num_cpus > 1) { - dev_err(card->dev, - "ASoC: multi cpu is not yet supported %s\n", - link->name); - return -EINVAL; - } + for_each_link_cpus(link, i, cpu) { + /* + * CPU device may be specified by either name or OF node, but + * can be left unspecified, and will be matched based on DAI + * name alone.. + */ + if (cpu->name && cpu->of_node) { + dev_err(card->dev, + "ASoC: Neither/both cpu name/of_node are set for %s\n", + link->name); + return -EINVAL; + } - /* - * CPU device may be specified by either name or OF node, but - * can be left unspecified, and will be matched based on DAI - * name alone.. - */ - if (link->cpus->name && link->cpus->of_node) { - dev_err(card->dev, - "ASoC: Neither/both cpu name/of_node are set for %s\n", - link->name); - return -EINVAL; - } + /* + * Defer card registration if cpu dai component is not added to + * component list. + */ + if ((cpu->of_node || cpu->name) && + !soc_find_component(cpu)) + return -EPROBE_DEFER; - /* - * Defer card registration if cpu dai component is not added to - * component list. - */ - if ((link->cpus->of_node || link->cpus->name) && - !soc_find_component(link->cpus)) - return -EPROBE_DEFER; - - /* - * At least one of CPU DAI name or CPU device name/node must be - * specified - */ - if (!link->cpus->dai_name && - !(link->cpus->name || link->cpus->of_node)) { - dev_err(card->dev, - "ASoC: Neither cpu_dai_name nor cpu_name/of_node are set for %s\n", - link->name); - return -EINVAL; + /* + * At least one of CPU DAI name or CPU device name/node must be + * specified + */ + if (!cpu->dai_name && + !(cpu->name || cpu->of_node)) { + dev_err(card->dev, + "ASoC: Neither cpu_dai_name nor cpu_name/of_node are set for %s\n", + link->name); + return -EINVAL; + } } return 0; @@ -966,7 +964,7 @@ int snd_soc_add_pcm_runtime(struct snd_soc_card *card, struct snd_soc_dai_link *dai_link) { struct snd_soc_pcm_runtime *rtd; - struct snd_soc_dai_link_component *codec, *platform; + struct snd_soc_dai_link_component *codec, *platform, *cpu; struct snd_soc_component *component; int i, ret; @@ -991,14 +989,19 @@ int snd_soc_add_pcm_runtime(struct snd_soc_card *card, if (!rtd) return -ENOMEM; - /* FIXME: we need multi CPU support in the future */ - rtd->cpu_dai = snd_soc_find_dai(dai_link->cpus); - if (!rtd->cpu_dai) { - dev_info(card->dev, "ASoC: CPU DAI %s not registered\n", - dai_link->cpus->dai_name); - goto _err_defer; + rtd->num_cpus = dai_link->num_cpus; + for_each_link_cpus(dai_link, i, cpu) { + rtd->cpu_dais[i] = snd_soc_find_dai(cpu); + if (!rtd->cpu_dais[i]) { + dev_info(card->dev, "ASoC: CPU DAI %s not registered\n", + cpu->dai_name); + goto _err_defer; + } + snd_soc_rtd_add_component(rtd, rtd->cpu_dais[i]->component); } - snd_soc_rtd_add_component(rtd, rtd->cpu_dai->component); + + /* Single cpu links expect cpu and cpu_dai in runtime data */ + rtd->cpu_dai = rtd->cpu_dais[0]; /* Find CODEC from registered CODECs */ rtd->num_codecs = dai_link->num_codecs; @@ -1034,20 +1037,20 @@ int snd_soc_add_pcm_runtime(struct snd_soc_card *card, } EXPORT_SYMBOL_GPL(snd_soc_add_pcm_runtime); -static int soc_dai_pcm_new(struct snd_soc_dai **dais, int num_dais, - struct snd_soc_pcm_runtime *rtd) +static int soc_dai_pcm_new(struct snd_soc_pcm_runtime *rtd) { + struct snd_soc_dai *dai; int i, ret = 0; - for (i = 0; i < num_dais; ++i) { - struct snd_soc_dai_driver *drv = dais[i]->driver; + for_each_rtd_dais(rtd, i, dai) { + struct snd_soc_dai_driver *drv = dai->driver; if (drv->pcm_new) - ret = drv->pcm_new(rtd, dais[i]); + ret = drv->pcm_new(rtd, dai); if (ret < 0) { - dev_err(dais[i]->dev, + dev_err(dai->dev, "ASoC: Failed to bind %s with pcm device\n", - dais[i]->name); + dai->name); return ret; } } @@ -1118,12 +1121,8 @@ static int soc_init_pcm_runtime(struct snd_soc_card *card, dai_link->stream_name, ret); return ret; } - ret = soc_dai_pcm_new(&cpu_dai, 1, rtd); - if (ret < 0) - return ret; - ret = soc_dai_pcm_new(rtd->codec_dais, - rtd->num_codecs, rtd); - return ret; + + return soc_dai_pcm_new(rtd); } static void soc_set_name_prefix(struct snd_soc_card *card, @@ -1256,8 +1255,18 @@ static int soc_probe_component(struct snd_soc_card *card, ret = snd_soc_dapm_add_routes(dapm, component->driver->dapm_routes, component->driver->num_dapm_routes); - if (ret < 0) - goto err_probe; + if (ret < 0) { + if (card->disable_route_checks) { + dev_info(card->dev, + "%s: disable_route_checks set, ignoring errors on add_routes\n", + __func__); + } else { + dev_err(card->dev, + "%s: snd_soc_dapm_add_routes failed: %d\n", + __func__, ret); + goto err_probe; + } + } /* see for_each_card_components */ list_add(&component->card_list, &card->component_dev_list); @@ -1309,24 +1318,22 @@ static int soc_probe_dai(struct snd_soc_dai *dai, int order) static void soc_remove_link_dais(struct snd_soc_card *card) { int i; - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; struct snd_soc_pcm_runtime *rtd; int order; for_each_comp_order(order) { for_each_card_rtds(card, rtd) { - /* remove the CODEC DAI */ - for_each_rtd_codec_dai(rtd, i, codec_dai) - soc_remove_dai(codec_dai, order); - - soc_remove_dai(rtd->cpu_dai, order); + /* remove DAIs */ + for_each_rtd_dais(rtd, i, dai) + soc_remove_dai(dai, order); } } } static int soc_probe_link_dais(struct snd_soc_card *card) { - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; struct snd_soc_pcm_runtime *rtd; int i, order, ret; @@ -1337,13 +1344,9 @@ static int soc_probe_link_dais(struct snd_soc_card *card) "ASoC: probe %s dai link %d late %d\n", card->name, rtd->num, order); - ret = soc_probe_dai(rtd->cpu_dai, order); - if (ret) - return ret; - - /* probe the CODEC DAI */ - for_each_rtd_codec_dai(rtd, i, codec_dai) { - ret = soc_probe_dai(codec_dai, order); + /* probe the CPU DAI */ + for_each_rtd_dais(rtd, i, dai) { + ret = soc_probe_dai(dai, order); if (ret) return ret; } @@ -1471,12 +1474,13 @@ static void soc_remove_aux_devices(struct snd_soc_card *card) int snd_soc_runtime_set_dai_fmt(struct snd_soc_pcm_runtime *rtd, unsigned int dai_fmt) { - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai; struct snd_soc_dai *codec_dai; + unsigned int inv_dai_fmt; unsigned int i; int ret; - for_each_rtd_codec_dai(rtd, i, codec_dai) { + for_each_rtd_codec_dais(rtd, i, codec_dai) { ret = snd_soc_dai_set_fmt(codec_dai, dai_fmt); if (ret != 0 && ret != -ENOTSUPP) { dev_warn(codec_dai->dev, @@ -1489,33 +1493,33 @@ int snd_soc_runtime_set_dai_fmt(struct snd_soc_pcm_runtime *rtd, * Flip the polarity for the "CPU" end of a CODEC<->CODEC link * the component which has non_legacy_dai_naming is Codec */ - if (cpu_dai->component->driver->non_legacy_dai_naming) { - unsigned int inv_dai_fmt; - - inv_dai_fmt = dai_fmt & ~SND_SOC_DAIFMT_MASTER_MASK; - switch (dai_fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBM_CFM: - inv_dai_fmt |= SND_SOC_DAIFMT_CBS_CFS; - break; - case SND_SOC_DAIFMT_CBM_CFS: - inv_dai_fmt |= SND_SOC_DAIFMT_CBS_CFM; - break; - case SND_SOC_DAIFMT_CBS_CFM: - inv_dai_fmt |= SND_SOC_DAIFMT_CBM_CFS; - break; - case SND_SOC_DAIFMT_CBS_CFS: - inv_dai_fmt |= SND_SOC_DAIFMT_CBM_CFM; - break; - } - - dai_fmt = inv_dai_fmt; + inv_dai_fmt = dai_fmt & ~SND_SOC_DAIFMT_MASTER_MASK; + switch (dai_fmt & SND_SOC_DAIFMT_MASTER_MASK) { + case SND_SOC_DAIFMT_CBM_CFM: + inv_dai_fmt |= SND_SOC_DAIFMT_CBS_CFS; + break; + case SND_SOC_DAIFMT_CBM_CFS: + inv_dai_fmt |= SND_SOC_DAIFMT_CBS_CFM; + break; + case SND_SOC_DAIFMT_CBS_CFM: + inv_dai_fmt |= SND_SOC_DAIFMT_CBM_CFS; + break; + case SND_SOC_DAIFMT_CBS_CFS: + inv_dai_fmt |= SND_SOC_DAIFMT_CBM_CFM; + break; } + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { + unsigned int fmt = dai_fmt; - ret = snd_soc_dai_set_fmt(cpu_dai, dai_fmt); - if (ret != 0 && ret != -ENOTSUPP) { - dev_warn(cpu_dai->dev, - "ASoC: Failed to set DAI format: %d\n", ret); - return ret; + if (cpu_dai->component->driver->non_legacy_dai_naming) + fmt = inv_dai_fmt; + + ret = snd_soc_dai_set_fmt(cpu_dai, fmt); + if (ret != 0 && ret != -ENOTSUPP) { + dev_warn(cpu_dai->dev, + "ASoC: Failed to set DAI format: %d\n", ret); + return ret; + } } return 0; @@ -1938,8 +1942,18 @@ static int snd_soc_bind_card(struct snd_soc_card *card) ret = snd_soc_dapm_add_routes(&card->dapm, card->dapm_routes, card->num_dapm_routes); - if (ret < 0) - goto probe_end; + if (ret < 0) { + if (card->disable_route_checks) { + dev_info(card->dev, + "%s: disable_route_checks set, ignoring errors on add_routes\n", + __func__); + } else { + dev_err(card->dev, + "%s: snd_soc_dapm_add_routes failed: %d\n", + __func__, ret); + goto probe_end; + } + } ret = snd_soc_dapm_add_routes(&card->dapm, card->of_dapm_routes, card->num_of_dapm_routes); @@ -3135,6 +3149,14 @@ int snd_soc_get_dai_name(struct of_phandle_args *args, *dai_name = dai->driver->name; if (!*dai_name) *dai_name = pos->name; + } else if (ret) { + /* + * if another error than ENOTSUPP is returned go on and + * check if another component is provided with the same + * node. This may happen if a device provides several + * components + */ + continue; } break; diff --git a/sound/soc/soc-dai.c b/sound/soc/soc-dai.c index 51031e330179..19142f6e533c 100644 --- a/sound/soc/soc-dai.c +++ b/sound/soc/soc-dai.c @@ -295,17 +295,24 @@ int snd_soc_dai_startup(struct snd_soc_dai *dai, { int ret = 0; - if (dai->driver->ops->startup) + if (!dai->started && + dai->driver->ops->startup) ret = dai->driver->ops->startup(substream, dai); + if (ret == 0) + dai->started = 1; + return ret; } void snd_soc_dai_shutdown(struct snd_soc_dai *dai, struct snd_pcm_substream *substream) { - if (dai->driver->ops->shutdown) + if (dai->started && + dai->driver->ops->shutdown) dai->driver->ops->shutdown(substream, dai); + + dai->started = 0; } int snd_soc_dai_prepare(struct snd_soc_dai *dai, @@ -383,12 +390,7 @@ int snd_soc_dai_compress_new(struct snd_soc_dai *dai, */ bool snd_soc_dai_stream_valid(struct snd_soc_dai *dai, int dir) { - struct snd_soc_pcm_stream *stream; - - if (dir == SNDRV_PCM_STREAM_PLAYBACK) - stream = &dai->driver->playback; - else - stream = &dai->driver->capture; + struct snd_soc_pcm_stream *stream = snd_soc_dai_get_pcm_stream(dai, dir); /* If the codec specifies any channels at all, it supports the stream */ return stream->channels_min; diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 9fb54e6fe254..04da7928c873 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -302,7 +302,7 @@ void dapm_mark_endpoints_dirty(struct snd_soc_card *card) mutex_lock(&card->dapm_mutex); - list_for_each_entry(w, &card->widgets, list) { + for_each_card_widgets(card, w) { if (w->is_ep) { dapm_mark_dirty(w, "Rechecking endpoints"); if (w->is_ep & SND_SOC_DAPM_EP_SINK) @@ -589,7 +589,7 @@ static void dapm_reset(struct snd_soc_card *card) memset(&card->dapm_stats, 0, sizeof(card->dapm_stats)); - list_for_each_entry(w, &card->widgets, list) { + for_each_card_widgets(card, w) { w->new_power = w->power; w->power_checked = false; } @@ -833,7 +833,7 @@ static int dapm_is_shared_kcontrol(struct snd_soc_dapm_context *dapm, *kcontrol = NULL; - list_for_each_entry(w, &dapm->card->widgets, list) { + for_each_card_widgets(dapm->card, w) { if (w == kcontrolw || w->dapm != kcontrolw->dapm) continue; for (i = 0; i < w->num_kcontrols; i++) { @@ -1105,6 +1105,11 @@ static int snd_soc_dapm_suspend_check(struct snd_soc_dapm_widget *widget) } } +static void dapm_widget_list_free(struct snd_soc_dapm_widget_list **list) +{ + kfree(*list); +} + static int dapm_widget_list_create(struct snd_soc_dapm_widget_list **list, struct list_head *widgets) { @@ -1310,6 +1315,11 @@ int snd_soc_dapm_dai_get_connected_widgets(struct snd_soc_dai *dai, int stream, return paths; } +void snd_soc_dapm_dai_free_widgets(struct snd_soc_dapm_widget_list **list) +{ + dapm_widget_list_free(list); +} + /* * Handler for regulator supply widget. */ @@ -1706,9 +1716,8 @@ static void dapm_seq_run(struct snd_soc_card *card, i, cur_subseq); } - list_for_each_entry(d, &card->dapm_list, list) { + for_each_card_dapms(card, d) soc_dapm_async_complete(d); - } } static void dapm_widget_update(struct snd_soc_card *card) @@ -1724,9 +1733,7 @@ static void dapm_widget_update(struct snd_soc_card *card) wlist = dapm_kcontrol_get_wlist(update->kcontrol); - for (wi = 0; wi < wlist->num_widgets; wi++) { - w = wlist->widgets[wi]; - + for_each_dapm_widgets(wlist, wi, w) { if (w->event && (w->event_flags & SND_SOC_DAPM_PRE_REG)) { ret = w->event(w, update->kcontrol, SND_SOC_DAPM_PRE_REG); if (ret != 0) @@ -1753,9 +1760,7 @@ static void dapm_widget_update(struct snd_soc_card *card) w->name, ret); } - for (wi = 0; wi < wlist->num_widgets; wi++) { - w = wlist->widgets[wi]; - + for_each_dapm_widgets(wlist, wi, w) { if (w->event && (w->event_flags & SND_SOC_DAPM_POST_REG)) { ret = w->event(w, update->kcontrol, SND_SOC_DAPM_POST_REG); if (ret != 0) @@ -1943,7 +1948,7 @@ static int dapm_power_widgets(struct snd_soc_card *card, int event) trace_snd_soc_dapm_start(card); - list_for_each_entry(d, &card->dapm_list, list) { + for_each_card_dapms(card, d) { if (dapm_idle_bias_off(d)) d->target_bias_level = SND_SOC_BIAS_OFF; else @@ -1962,7 +1967,7 @@ static int dapm_power_widgets(struct snd_soc_card *card, int event) dapm_power_one_widget(w, &up_list, &down_list); } - list_for_each_entry(w, &card->widgets, list) { + for_each_card_widgets(card, w) { switch (w->id) { case snd_soc_dapm_pre: case snd_soc_dapm_post: @@ -2007,10 +2012,10 @@ static int dapm_power_widgets(struct snd_soc_card *card, int event) * they're not ground referenced. */ bias = SND_SOC_BIAS_OFF; - list_for_each_entry(d, &card->dapm_list, list) + for_each_card_dapms(card, d) if (d->target_bias_level > bias) bias = d->target_bias_level; - list_for_each_entry(d, &card->dapm_list, list) + for_each_card_dapms(card, d) if (!dapm_idle_bias_off(d)) d->target_bias_level = bias; @@ -2019,7 +2024,7 @@ static int dapm_power_widgets(struct snd_soc_card *card, int event) /* Run card bias changes at first */ dapm_pre_sequence_async(&card->dapm, 0); /* Run other bias changes in parallel */ - list_for_each_entry(d, &card->dapm_list, list) { + for_each_card_dapms(card, d) { if (d != &card->dapm && d->bias_level != d->target_bias_level) async_schedule_domain(dapm_pre_sequence_async, d, &async_domain); @@ -2043,7 +2048,7 @@ static int dapm_power_widgets(struct snd_soc_card *card, int event) dapm_seq_run(card, &up_list, event, true); /* Run all the bias changes in parallel */ - list_for_each_entry(d, &card->dapm_list, list) { + for_each_card_dapms(card, d) { if (d != &card->dapm && d->bias_level != d->target_bias_level) async_schedule_domain(dapm_post_sequence_async, d, &async_domain); @@ -2053,7 +2058,7 @@ static int dapm_power_widgets(struct snd_soc_card *card, int event) dapm_post_sequence_async(&card->dapm, 0); /* do we need to notify any clients that DAPM event is complete */ - list_for_each_entry(d, &card->dapm_list, list) { + for_each_card_dapms(card, d) { if (!d->component) continue; @@ -2286,7 +2291,7 @@ int snd_soc_dapm_mux_update_power(struct snd_soc_dapm_context *dapm, card->update = NULL; mutex_unlock(&card->dapm_mutex); if (ret > 0) - soc_dpcm_runtime_update(card); + snd_soc_dpcm_runtime_update(card); return ret; } EXPORT_SYMBOL_GPL(snd_soc_dapm_mux_update_power); @@ -2351,7 +2356,7 @@ int snd_soc_dapm_mixer_update_power(struct snd_soc_dapm_context *dapm, card->update = NULL; mutex_unlock(&card->dapm_mutex); if (ret > 0) - soc_dpcm_runtime_update(card); + snd_soc_dpcm_runtime_update(card); return ret; } EXPORT_SYMBOL_GPL(snd_soc_dapm_mixer_update_power); @@ -2371,7 +2376,7 @@ static ssize_t dapm_widget_show_component(struct snd_soc_component *cmpnt, if (!cmpnt->card) return 0; - list_for_each_entry(w, &cmpnt->card->widgets, list) { + for_each_card_widgets(cmpnt->card, w) { if (w->dapm != dapm) continue; @@ -2431,7 +2436,7 @@ static ssize_t dapm_widget_show(struct device *dev, mutex_lock(&rtd->card->dapm_mutex); - for_each_rtd_codec_dai(rtd, i, codec_dai) { + for_each_rtd_codec_dais(rtd, i, codec_dai) { struct snd_soc_component *cmpnt = codec_dai->component; count += dapm_widget_show_component(cmpnt, buf + count); @@ -2491,7 +2496,7 @@ static void dapm_free_widgets(struct snd_soc_dapm_context *dapm) { struct snd_soc_dapm_widget *w, *next_w; - list_for_each_entry_safe(w, next_w, &dapm->card->widgets, list) { + for_each_card_widgets_safe(dapm->card, w, next_w) { if (w->dapm != dapm) continue; snd_soc_dapm_free_widget(w); @@ -2506,7 +2511,7 @@ static struct snd_soc_dapm_widget *dapm_find_widget( struct snd_soc_dapm_widget *w; struct snd_soc_dapm_widget *fallback = NULL; - list_for_each_entry(w, &dapm->card->widgets, list) { + for_each_card_widgets(dapm->card, w) { if (!strcmp(w->name, pin)) { if (w->dapm == dapm) return w; @@ -2624,10 +2629,7 @@ static int dapm_update_dai_unlocked(struct snd_pcm_substream *substream, struct snd_soc_dapm_widget *w; int ret; - if (dir == SNDRV_PCM_STREAM_PLAYBACK) - w = dai->playback_widget; - else - w = dai->capture_widget; + w = snd_soc_dai_get_widget(dai, dir); if (!w) return 0; @@ -2908,7 +2910,7 @@ static int snd_soc_dapm_add_route(struct snd_soc_dapm_context *dapm, * find src and dest widgets over all widgets but favor a widget from * current DAPM context */ - list_for_each_entry(w, &dapm->card->widgets, list) { + for_each_card_widgets(dapm->card, w) { if (!wsink && !(strcmp(w->name, sink))) { wtsink = w; if (w->dapm == dapm) { @@ -3187,7 +3189,7 @@ int snd_soc_dapm_new_widgets(struct snd_soc_card *card) mutex_lock_nested(&card->dapm_mutex, SND_SOC_DAPM_CLASS_INIT); - list_for_each_entry(w, &card->widgets, list) + for_each_card_widgets(card, w) { if (w->new) continue; @@ -3394,7 +3396,7 @@ int snd_soc_dapm_put_volsw(struct snd_kcontrol *kcontrol, mutex_unlock(&card->dapm_mutex); if (ret > 0) - soc_dpcm_runtime_update(card); + snd_soc_dpcm_runtime_update(card); return change; } @@ -3499,7 +3501,7 @@ int snd_soc_dapm_put_enum_double(struct snd_kcontrol *kcontrol, mutex_unlock(&card->dapm_mutex); if (ret > 0) - soc_dpcm_runtime_update(card); + snd_soc_dpcm_runtime_update(card); return change; } @@ -3604,6 +3606,9 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm, ret = PTR_ERR(w->pinctrl); goto request_failed; } + + /* set to sleep_state when initializing */ + dapm_pinctrl_event(w, NULL, SND_SOC_DAPM_POST_PMD); break; case snd_soc_dapm_clock_supply: w->clk = devm_clk_get(dapm->dev, w->name); @@ -3698,6 +3703,7 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm, w->dapm = dapm; INIT_LIST_HEAD(&w->list); INIT_LIST_HEAD(&w->dirty); + /* see for_each_card_widgets */ list_add_tail(&w->list, &dapm->card->widgets); snd_soc_dapm_for_each_direction(dir) { @@ -4222,7 +4228,7 @@ int snd_soc_dapm_link_dai_widgets(struct snd_soc_card *card) struct snd_soc_dai *dai; /* For each DAI widget... */ - list_for_each_entry(dai_w, &card->widgets, list) { + for_each_card_widgets(card, dai_w) { switch (dai_w->id) { case snd_soc_dapm_dai_in: case snd_soc_dapm_dai_out: @@ -4241,7 +4247,7 @@ int snd_soc_dapm_link_dai_widgets(struct snd_soc_card *card) dai = dai_w->priv; /* ...find all widgets with the same stream and link them */ - list_for_each_entry(w, &card->widgets, list) { + for_each_card_widgets(card, w) { if (w->dapm != dai_w->dapm) continue; @@ -4271,16 +4277,15 @@ int snd_soc_dapm_link_dai_widgets(struct snd_soc_card *card) return 0; } -static void dapm_connect_dai_link_widgets(struct snd_soc_card *card, - struct snd_soc_pcm_runtime *rtd) +static void dapm_add_valid_dai_widget(struct snd_soc_card *card, + struct snd_soc_pcm_runtime *rtd, + struct snd_soc_dai *codec_dai, + struct snd_soc_dai *cpu_dai) { - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; struct snd_soc_dapm_widget *playback = NULL, *capture = NULL; struct snd_soc_dapm_widget *codec, *playback_cpu, *capture_cpu; struct snd_pcm_substream *substream; struct snd_pcm_str *streams = rtd->pcm->streams; - int i; if (rtd->dai_link->params) { playback_cpu = cpu_dai->capture_widget; @@ -4292,77 +4297,92 @@ static void dapm_connect_dai_link_widgets(struct snd_soc_card *card, capture_cpu = capture; } - for_each_rtd_codec_dai(rtd, i, codec_dai) { - /* connect BE DAI playback if widgets are valid */ - codec = codec_dai->playback_widget; + /* connect BE DAI playback if widgets are valid */ + codec = codec_dai->playback_widget; - if (playback_cpu && codec) { - if (!playback) { - substream = streams[SNDRV_PCM_STREAM_PLAYBACK].substream; - playback = snd_soc_dapm_new_dai(card, substream, - "playback"); - if (IS_ERR(playback)) { - dev_err(rtd->dev, - "ASoC: Failed to create DAI %s: %ld\n", - codec_dai->name, - PTR_ERR(playback)); - continue; - } - - snd_soc_dapm_add_path(&card->dapm, playback_cpu, - playback, NULL, NULL); + if (playback_cpu && codec) { + if (!playback) { + substream = streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + playback = snd_soc_dapm_new_dai(card, substream, + "playback"); + if (IS_ERR(playback)) { + dev_err(rtd->dev, + "ASoC: Failed to create DAI %s: %ld\n", + codec_dai->name, + PTR_ERR(playback)); + goto capture; } - dev_dbg(rtd->dev, "connected DAI link %s:%s -> %s:%s\n", - cpu_dai->component->name, playback_cpu->name, - codec_dai->component->name, codec->name); - - snd_soc_dapm_add_path(&card->dapm, playback, codec, - NULL, NULL); + snd_soc_dapm_add_path(&card->dapm, playback_cpu, + playback, NULL, NULL); } + + dev_dbg(rtd->dev, "connected DAI link %s:%s -> %s:%s\n", + cpu_dai->component->name, playback_cpu->name, + codec_dai->component->name, codec->name); + + snd_soc_dapm_add_path(&card->dapm, playback, codec, + NULL, NULL); } - for_each_rtd_codec_dai(rtd, i, codec_dai) { - /* connect BE DAI capture if widgets are valid */ - codec = codec_dai->capture_widget; +capture: + /* connect BE DAI capture if widgets are valid */ + codec = codec_dai->capture_widget; - if (codec && capture_cpu) { - if (!capture) { - substream = streams[SNDRV_PCM_STREAM_CAPTURE].substream; - capture = snd_soc_dapm_new_dai(card, substream, - "capture"); - if (IS_ERR(capture)) { - dev_err(rtd->dev, - "ASoC: Failed to create DAI %s: %ld\n", - codec_dai->name, - PTR_ERR(capture)); - continue; - } - - snd_soc_dapm_add_path(&card->dapm, capture, - capture_cpu, NULL, NULL); + if (codec && capture_cpu) { + if (!capture) { + substream = streams[SNDRV_PCM_STREAM_CAPTURE].substream; + capture = snd_soc_dapm_new_dai(card, substream, + "capture"); + if (IS_ERR(capture)) { + dev_err(rtd->dev, + "ASoC: Failed to create DAI %s: %ld\n", + codec_dai->name, + PTR_ERR(capture)); + return; } - dev_dbg(rtd->dev, "connected DAI link %s:%s -> %s:%s\n", - codec_dai->component->name, codec->name, - cpu_dai->component->name, capture_cpu->name); - - snd_soc_dapm_add_path(&card->dapm, codec, capture, - NULL, NULL); + snd_soc_dapm_add_path(&card->dapm, capture, + capture_cpu, NULL, NULL); } + + dev_dbg(rtd->dev, "connected DAI link %s:%s -> %s:%s\n", + codec_dai->component->name, codec->name, + cpu_dai->component->name, capture_cpu->name); + + snd_soc_dapm_add_path(&card->dapm, codec, capture, + NULL, NULL); } } +static void dapm_connect_dai_link_widgets(struct snd_soc_card *card, + struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_dai *codec_dai; + int i; + + if (rtd->num_cpus == 1) { + for_each_rtd_codec_dais(rtd, i, codec_dai) + dapm_add_valid_dai_widget(card, rtd, codec_dai, + rtd->cpu_dais[0]); + } else if (rtd->num_codecs == rtd->num_cpus) { + for_each_rtd_codec_dais(rtd, i, codec_dai) + dapm_add_valid_dai_widget(card, rtd, codec_dai, + rtd->cpu_dais[i]); + } else { + dev_err(card->dev, + "N cpus to M codecs link is not supported yet\n"); + } + +} + static void soc_dapm_dai_stream_event(struct snd_soc_dai *dai, int stream, int event) { struct snd_soc_dapm_widget *w; unsigned int ep; - if (stream == SNDRV_PCM_STREAM_PLAYBACK) - w = dai->playback_widget; - else - w = dai->capture_widget; + w = snd_soc_dai_get_widget(dai, stream); if (w) { dapm_mark_dirty(w, "stream event"); @@ -4413,12 +4433,11 @@ void snd_soc_dapm_connect_dai_link_widgets(struct snd_soc_card *card) static void soc_dapm_stream_event(struct snd_soc_pcm_runtime *rtd, int stream, int event) { - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; int i; - soc_dapm_dai_stream_event(rtd->cpu_dai, stream, event); - for_each_rtd_codec_dai(rtd, i, codec_dai) - soc_dapm_dai_stream_event(codec_dai, stream, event); + for_each_rtd_dais(rtd, i, dai) + soc_dapm_dai_stream_event(dai, stream, event); dapm_power_widgets(rtd->card, event); } @@ -4754,6 +4773,7 @@ void snd_soc_dapm_init(struct snd_soc_dapm_context *dapm, } INIT_LIST_HEAD(&dapm->list); + /* see for_each_card_dapms */ list_add(&dapm->list, &card->dapm_list); } EXPORT_SYMBOL_GPL(snd_soc_dapm_init); @@ -4767,7 +4787,7 @@ static void soc_dapm_shutdown_dapm(struct snd_soc_dapm_context *dapm) mutex_lock(&card->dapm_mutex); - list_for_each_entry(w, &dapm->card->widgets, list) { + for_each_card_widgets(dapm->card, w) { if (w->dapm != dapm) continue; if (w->power) { @@ -4800,7 +4820,7 @@ void snd_soc_dapm_shutdown(struct snd_soc_card *card) { struct snd_soc_dapm_context *dapm; - list_for_each_entry(dapm, &card->dapm_list, list) { + for_each_card_dapms(card, dapm) { if (dapm != &card->dapm) { soc_dapm_shutdown_dapm(dapm); if (dapm->bias_level == SND_SOC_BIAS_STANDBY) diff --git a/sound/soc/soc-generic-dmaengine-pcm.c b/sound/soc/soc-generic-dmaengine-pcm.c index 2cc25651661c..facf1922a714 100644 --- a/sound/soc/soc-generic-dmaengine-pcm.c +++ b/sound/soc/soc-generic-dmaengine-pcm.c @@ -62,6 +62,12 @@ int snd_dmaengine_pcm_prepare_slave_config(struct snd_pcm_substream *substream, struct snd_dmaengine_dai_dma_data *dma_data; int ret; + if (rtd->num_cpus > 1) { + dev_err(rtd->dev, + "%s doesn't support Multi CPU yet\n", __func__); + return -EINVAL; + } + dma_data = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); ret = snd_hwparams_to_dma_slave_config(substream, params, slave_config); @@ -118,6 +124,12 @@ dmaengine_pcm_set_runtime_hwparams(struct snd_soc_component *component, struct snd_dmaengine_dai_dma_data *dma_data; struct snd_pcm_hardware hw; + if (rtd->num_cpus > 1) { + dev_err(rtd->dev, + "%s doesn't support Multi CPU yet\n", __func__); + return -EINVAL; + } + if (pcm->config && pcm->config->pcm_hardware) return snd_soc_set_runtime_hwparams(substream, pcm->config->pcm_hardware); @@ -185,6 +197,12 @@ static struct dma_chan *dmaengine_pcm_compat_request_channel( struct snd_dmaengine_dai_dma_data *dma_data; dma_filter_fn fn = NULL; + if (rtd->num_cpus > 1) { + dev_err(rtd->dev, + "%s doesn't support Multi CPU yet\n", __func__); + return NULL; + } + dma_data = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); if ((pcm->flags & SND_DMAENGINE_PCM_FLAG_HALF_DUPLEX) && pcm->chan[0]) @@ -237,7 +255,7 @@ static int dmaengine_pcm_new(struct snd_soc_component *component, max_buffer_size = SIZE_MAX; } - for (i = SNDRV_PCM_STREAM_PLAYBACK; i <= SNDRV_PCM_STREAM_CAPTURE; i++) { + for_each_pcm_streams(i) { substream = rtd->pcm->streams[i].substream; if (!substream) continue; @@ -371,8 +389,7 @@ static int dmaengine_pcm_request_chan_of(struct dmaengine_pcm *pcm, dev = config->dma_dev; } - for (i = SNDRV_PCM_STREAM_PLAYBACK; i <= SNDRV_PCM_STREAM_CAPTURE; - i++) { + for_each_pcm_streams(i) { if (pcm->flags & SND_DMAENGINE_PCM_FLAG_HALF_DUPLEX) name = "rx-tx"; else @@ -401,8 +418,7 @@ static void dmaengine_pcm_release_chan(struct dmaengine_pcm *pcm) { unsigned int i; - for (i = SNDRV_PCM_STREAM_PLAYBACK; i <= SNDRV_PCM_STREAM_CAPTURE; - i++) { + for_each_pcm_streams(i) { if (!pcm->chan[i]) continue; dma_release_channel(pcm->chan[i]); diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c index dac324bdcd19..da138de15358 100644 --- a/sound/soc/soc-pcm.c +++ b/sound/soc/soc-pcm.c @@ -28,6 +28,180 @@ #define DPCM_MAX_BE_USERS 8 +#ifdef CONFIG_DEBUG_FS +static const char *dpcm_state_string(enum snd_soc_dpcm_state state) +{ + switch (state) { + case SND_SOC_DPCM_STATE_NEW: + return "new"; + case SND_SOC_DPCM_STATE_OPEN: + return "open"; + case SND_SOC_DPCM_STATE_HW_PARAMS: + return "hw_params"; + case SND_SOC_DPCM_STATE_PREPARE: + return "prepare"; + case SND_SOC_DPCM_STATE_START: + return "start"; + case SND_SOC_DPCM_STATE_STOP: + return "stop"; + case SND_SOC_DPCM_STATE_SUSPEND: + return "suspend"; + case SND_SOC_DPCM_STATE_PAUSED: + return "paused"; + case SND_SOC_DPCM_STATE_HW_FREE: + return "hw_free"; + case SND_SOC_DPCM_STATE_CLOSE: + return "close"; + } + + return "unknown"; +} + +static ssize_t dpcm_show_state(struct snd_soc_pcm_runtime *fe, + int stream, char *buf, size_t size) +{ + struct snd_pcm_hw_params *params = &fe->dpcm[stream].hw_params; + struct snd_soc_dpcm *dpcm; + ssize_t offset = 0; + unsigned long flags; + + /* FE state */ + offset += scnprintf(buf + offset, size - offset, + "[%s - %s]\n", fe->dai_link->name, + stream ? "Capture" : "Playback"); + + offset += scnprintf(buf + offset, size - offset, "State: %s\n", + dpcm_state_string(fe->dpcm[stream].state)); + + if ((fe->dpcm[stream].state >= SND_SOC_DPCM_STATE_HW_PARAMS) && + (fe->dpcm[stream].state <= SND_SOC_DPCM_STATE_STOP)) + offset += scnprintf(buf + offset, size - offset, + "Hardware Params: " + "Format = %s, Channels = %d, Rate = %d\n", + snd_pcm_format_name(params_format(params)), + params_channels(params), + params_rate(params)); + + /* BEs state */ + offset += scnprintf(buf + offset, size - offset, "Backends:\n"); + + if (list_empty(&fe->dpcm[stream].be_clients)) { + offset += scnprintf(buf + offset, size - offset, + " No active DSP links\n"); + goto out; + } + + spin_lock_irqsave(&fe->card->dpcm_lock, flags); + for_each_dpcm_be(fe, stream, dpcm) { + struct snd_soc_pcm_runtime *be = dpcm->be; + params = &dpcm->hw_params; + + offset += scnprintf(buf + offset, size - offset, + "- %s\n", be->dai_link->name); + + offset += scnprintf(buf + offset, size - offset, + " State: %s\n", + dpcm_state_string(be->dpcm[stream].state)); + + if ((be->dpcm[stream].state >= SND_SOC_DPCM_STATE_HW_PARAMS) && + (be->dpcm[stream].state <= SND_SOC_DPCM_STATE_STOP)) + offset += scnprintf(buf + offset, size - offset, + " Hardware Params: " + "Format = %s, Channels = %d, Rate = %d\n", + snd_pcm_format_name(params_format(params)), + params_channels(params), + params_rate(params)); + } + spin_unlock_irqrestore(&fe->card->dpcm_lock, flags); +out: + return offset; +} + +static ssize_t dpcm_state_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct snd_soc_pcm_runtime *fe = file->private_data; + ssize_t out_count = PAGE_SIZE, offset = 0, ret = 0; + int stream; + char *buf; + + if (fe->num_cpus > 1) { + dev_err(fe->dev, + "%s doesn't support Multi CPU yet\n", __func__); + return -EINVAL; + } + + buf = kmalloc(out_count, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + for_each_pcm_streams(stream) + if (snd_soc_dai_stream_valid(fe->cpu_dai, stream)) + offset += dpcm_show_state(fe, stream, + buf + offset, + out_count - offset); + + ret = simple_read_from_buffer(user_buf, count, ppos, buf, offset); + + kfree(buf); + return ret; +} + +static const struct file_operations dpcm_state_fops = { + .open = simple_open, + .read = dpcm_state_read_file, + .llseek = default_llseek, +}; + +void soc_dpcm_debugfs_add(struct snd_soc_pcm_runtime *rtd) +{ + if (!rtd->dai_link) + return; + + if (!rtd->dai_link->dynamic) + return; + + if (!rtd->card->debugfs_card_root) + return; + + rtd->debugfs_dpcm_root = debugfs_create_dir(rtd->dai_link->name, + rtd->card->debugfs_card_root); + + debugfs_create_file("state", 0444, rtd->debugfs_dpcm_root, + rtd, &dpcm_state_fops); +} + +static void dpcm_create_debugfs_state(struct snd_soc_dpcm *dpcm, int stream) +{ + char *name; + + name = kasprintf(GFP_KERNEL, "%s:%s", dpcm->be->dai_link->name, + stream ? "capture" : "playback"); + if (name) { + dpcm->debugfs_state = debugfs_create_dir( + name, dpcm->fe->debugfs_dpcm_root); + debugfs_create_u32("state", 0644, dpcm->debugfs_state, + &dpcm->state); + kfree(name); + } +} + +static void dpcm_remove_debugfs_state(struct snd_soc_dpcm *dpcm) +{ + debugfs_remove_recursive(dpcm->debugfs_state); +} + +#else +static inline void dpcm_create_debugfs_state(struct snd_soc_dpcm *dpcm, + int stream) +{ +} + +static inline void dpcm_remove_debugfs_state(struct snd_soc_dpcm *dpcm) +{ +} +#endif + static int soc_rtd_startup(struct snd_soc_pcm_runtime *rtd, struct snd_pcm_substream *substream) { @@ -82,6 +256,21 @@ static int soc_rtd_trigger(struct snd_soc_pcm_runtime *rtd, return 0; } +static void snd_soc_runtime_action(struct snd_soc_pcm_runtime *rtd, + int stream, int action) +{ + struct snd_soc_dai *dai; + int i; + + lockdep_assert_held(&rtd->card->pcm_mutex); + + for_each_rtd_dais(rtd, i, dai) { + dai->stream_active[stream] += action; + dai->active += action; + dai->component->active += action; + } +} + /** * snd_soc_runtime_activate() - Increment active count for PCM runtime components * @rtd: ASoC PCM runtime that is activated @@ -94,29 +283,9 @@ static int soc_rtd_trigger(struct snd_soc_pcm_runtime *rtd, */ void snd_soc_runtime_activate(struct snd_soc_pcm_runtime *rtd, int stream) { - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; - int i; - - lockdep_assert_held(&rtd->card->pcm_mutex); - - if (stream == SNDRV_PCM_STREAM_PLAYBACK) { - cpu_dai->playback_active++; - for_each_rtd_codec_dai(rtd, i, codec_dai) - codec_dai->playback_active++; - } else { - cpu_dai->capture_active++; - for_each_rtd_codec_dai(rtd, i, codec_dai) - codec_dai->capture_active++; - } - - cpu_dai->active++; - cpu_dai->component->active++; - for_each_rtd_codec_dai(rtd, i, codec_dai) { - codec_dai->active++; - codec_dai->component->active++; - } + snd_soc_runtime_action(rtd, stream, 1); } +EXPORT_SYMBOL_GPL(snd_soc_runtime_activate); /** * snd_soc_runtime_deactivate() - Decrement active count for PCM runtime components @@ -130,29 +299,9 @@ void snd_soc_runtime_activate(struct snd_soc_pcm_runtime *rtd, int stream) */ void snd_soc_runtime_deactivate(struct snd_soc_pcm_runtime *rtd, int stream) { - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; - int i; - - lockdep_assert_held(&rtd->card->pcm_mutex); - - if (stream == SNDRV_PCM_STREAM_PLAYBACK) { - cpu_dai->playback_active--; - for_each_rtd_codec_dai(rtd, i, codec_dai) - codec_dai->playback_active--; - } else { - cpu_dai->capture_active--; - for_each_rtd_codec_dai(rtd, i, codec_dai) - codec_dai->capture_active--; - } - - cpu_dai->active--; - cpu_dai->component->active--; - for_each_rtd_codec_dai(rtd, i, codec_dai) { - codec_dai->component->active--; - codec_dai->active--; - } + snd_soc_runtime_action(rtd, stream, -1); } +EXPORT_SYMBOL_GPL(snd_soc_runtime_deactivate); /** * snd_soc_runtime_ignore_pmdown_time() - Check whether to ignore the power down delay @@ -287,8 +436,8 @@ static int soc_pcm_params_symmetry(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; + struct snd_soc_dai *cpu_dai; unsigned int rate, channels, sample_bits, symmetry, i; rate = params_rate(params); @@ -296,40 +445,51 @@ static int soc_pcm_params_symmetry(struct snd_pcm_substream *substream, sample_bits = snd_pcm_format_physical_width(params_format(params)); /* reject unmatched parameters when applying symmetry */ - symmetry = cpu_dai->driver->symmetric_rates || - rtd->dai_link->symmetric_rates; + symmetry = rtd->dai_link->symmetric_rates; - for_each_rtd_codec_dai(rtd, i, codec_dai) - symmetry |= codec_dai->driver->symmetric_rates; + for_each_rtd_cpu_dais(rtd, i, dai) + symmetry |= dai->driver->symmetric_rates; - if (symmetry && cpu_dai->rate && cpu_dai->rate != rate) { - dev_err(rtd->dev, "ASoC: unmatched rate symmetry: %d - %d\n", - cpu_dai->rate, rate); - return -EINVAL; + if (symmetry) { + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { + if (cpu_dai->rate && cpu_dai->rate != rate) { + dev_err(rtd->dev, "ASoC: unmatched rate symmetry: %d - %d\n", + cpu_dai->rate, rate); + return -EINVAL; + } + } } - symmetry = cpu_dai->driver->symmetric_channels || - rtd->dai_link->symmetric_channels; + symmetry = rtd->dai_link->symmetric_channels; - for_each_rtd_codec_dai(rtd, i, codec_dai) - symmetry |= codec_dai->driver->symmetric_channels; + for_each_rtd_dais(rtd, i, dai) + symmetry |= dai->driver->symmetric_channels; - if (symmetry && cpu_dai->channels && cpu_dai->channels != channels) { - dev_err(rtd->dev, "ASoC: unmatched channel symmetry: %d - %d\n", - cpu_dai->channels, channels); - return -EINVAL; + if (symmetry) { + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { + if (cpu_dai->channels && + cpu_dai->channels != channels) { + dev_err(rtd->dev, "ASoC: unmatched channel symmetry: %d - %d\n", + cpu_dai->channels, channels); + return -EINVAL; + } + } } - symmetry = cpu_dai->driver->symmetric_samplebits || - rtd->dai_link->symmetric_samplebits; + symmetry = rtd->dai_link->symmetric_samplebits; - for_each_rtd_codec_dai(rtd, i, codec_dai) - symmetry |= codec_dai->driver->symmetric_samplebits; + for_each_rtd_dais(rtd, i, dai) + symmetry |= dai->driver->symmetric_samplebits; - if (symmetry && cpu_dai->sample_bits && cpu_dai->sample_bits != sample_bits) { - dev_err(rtd->dev, "ASoC: unmatched sample bits symmetry: %d - %d\n", - cpu_dai->sample_bits, sample_bits); - return -EINVAL; + if (symmetry) { + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { + if (cpu_dai->sample_bits && + cpu_dai->sample_bits != sample_bits) { + dev_err(rtd->dev, "ASoC: unmatched sample bits symmetry: %d - %d\n", + cpu_dai->sample_bits, sample_bits); + return -EINVAL; + } + } } return 0; @@ -338,20 +498,19 @@ static int soc_pcm_params_symmetry(struct snd_pcm_substream *substream, static bool soc_pcm_has_symmetry(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai_driver *cpu_driver = rtd->cpu_dai->driver; struct snd_soc_dai_link *link = rtd->dai_link; - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; unsigned int symmetry, i; - symmetry = cpu_driver->symmetric_rates || link->symmetric_rates || - cpu_driver->symmetric_channels || link->symmetric_channels || - cpu_driver->symmetric_samplebits || link->symmetric_samplebits; + symmetry = link->symmetric_rates || + link->symmetric_channels || + link->symmetric_samplebits; - for_each_rtd_codec_dai(rtd, i, codec_dai) + for_each_rtd_dais(rtd, i, dai) symmetry = symmetry || - codec_dai->driver->symmetric_rates || - codec_dai->driver->symmetric_channels || - codec_dai->driver->symmetric_samplebits; + dai->driver->symmetric_rates || + dai->driver->symmetric_channels || + dai->driver->symmetric_samplebits; return symmetry; } @@ -373,77 +532,98 @@ static void soc_pcm_set_msb(struct snd_pcm_substream *substream, int bits) static void soc_pcm_apply_msb(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai; struct snd_soc_dai *codec_dai; + struct snd_soc_pcm_stream *pcm_codec, *pcm_cpu; + int stream = substream->stream; int i; - unsigned int bits = 0, cpu_bits; + unsigned int bits = 0, cpu_bits = 0; - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - for_each_rtd_codec_dai(rtd, i, codec_dai) { - if (codec_dai->driver->playback.sig_bits == 0) { - bits = 0; - break; - } - bits = max(codec_dai->driver->playback.sig_bits, bits); + for_each_rtd_codec_dais(rtd, i, codec_dai) { + pcm_codec = snd_soc_dai_get_pcm_stream(codec_dai, stream); + + if (pcm_codec->sig_bits == 0) { + bits = 0; + break; } - cpu_bits = cpu_dai->driver->playback.sig_bits; - } else { - for_each_rtd_codec_dai(rtd, i, codec_dai) { - if (codec_dai->driver->capture.sig_bits == 0) { - bits = 0; - break; - } - bits = max(codec_dai->driver->capture.sig_bits, bits); + bits = max(pcm_codec->sig_bits, bits); + } + + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { + pcm_cpu = snd_soc_dai_get_pcm_stream(cpu_dai, stream); + + if (pcm_cpu->sig_bits == 0) { + cpu_bits = 0; + break; } - cpu_bits = cpu_dai->driver->capture.sig_bits; + cpu_bits = max(pcm_cpu->sig_bits, cpu_bits); } soc_pcm_set_msb(substream, bits); soc_pcm_set_msb(substream, cpu_bits); } -static void soc_pcm_init_runtime_hw(struct snd_pcm_substream *substream) +/** + * snd_soc_runtime_calc_hw() - Calculate hw limits for a PCM stream + * @rtd: ASoC PCM runtime + * @hw: PCM hardware parameters (output) + * @stream: Direction of the PCM stream + * + * Calculates the subset of stream parameters supported by all DAIs + * associated with the PCM stream. + */ +int snd_soc_runtime_calc_hw(struct snd_soc_pcm_runtime *rtd, + struct snd_pcm_hardware *hw, int stream) { - struct snd_pcm_runtime *runtime = substream->runtime; - struct snd_pcm_hardware *hw = &runtime->hw; - struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_dai *codec_dai; - struct snd_soc_dai_driver *cpu_dai_drv = rtd->cpu_dai->driver; - struct snd_soc_dai_driver *codec_dai_drv; + struct snd_soc_dai *cpu_dai; struct snd_soc_pcm_stream *codec_stream; struct snd_soc_pcm_stream *cpu_stream; unsigned int chan_min = 0, chan_max = UINT_MAX; + unsigned int cpu_chan_min = 0, cpu_chan_max = UINT_MAX; unsigned int rate_min = 0, rate_max = UINT_MAX; - unsigned int rates = UINT_MAX; + unsigned int cpu_rate_min = 0, cpu_rate_max = UINT_MAX; + unsigned int rates = UINT_MAX, cpu_rates = UINT_MAX; u64 formats = ULLONG_MAX; int i; - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) - cpu_stream = &cpu_dai_drv->playback; - else - cpu_stream = &cpu_dai_drv->capture; + /* first calculate min/max only for CPUs in the DAI link */ + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { - /* first calculate min/max only for CODECs in the DAI link */ - for_each_rtd_codec_dai(rtd, i, codec_dai) { + /* + * Skip CPUs which don't support the current stream type. + * Otherwise, since the rate, channel, and format values will + * zero in that case, we would have no usable settings left, + * causing the resulting setup to fail. + */ + if (!snd_soc_dai_stream_valid(cpu_dai, stream)) + continue; + + cpu_stream = snd_soc_dai_get_pcm_stream(cpu_dai, stream); + + cpu_chan_min = max(cpu_chan_min, cpu_stream->channels_min); + cpu_chan_max = min(cpu_chan_max, cpu_stream->channels_max); + cpu_rate_min = max(cpu_rate_min, cpu_stream->rate_min); + cpu_rate_max = min_not_zero(cpu_rate_max, cpu_stream->rate_max); + formats &= cpu_stream->formats; + cpu_rates = snd_pcm_rate_mask_intersect(cpu_stream->rates, + cpu_rates); + } + + /* second calculate min/max only for CODECs in the DAI link */ + for_each_rtd_codec_dais(rtd, i, codec_dai) { /* * Skip CODECs which don't support the current stream type. * Otherwise, since the rate, channel, and format values will * zero in that case, we would have no usable settings left, * causing the resulting setup to fail. - * At least one CODEC should match, otherwise we should have - * bailed out on a higher level, since there would be no - * CODEC to support the transfer direction in that case. */ - if (!snd_soc_dai_stream_valid(codec_dai, - substream->stream)) + if (!snd_soc_dai_stream_valid(codec_dai, stream)) continue; - codec_dai_drv = codec_dai->driver; - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) - codec_stream = &codec_dai_drv->playback; - else - codec_stream = &codec_dai_drv->capture; + codec_stream = snd_soc_dai_get_pcm_stream(codec_dai, stream); + chan_min = max(chan_min, codec_stream->channels_min); chan_max = min(chan_max, codec_stream->channels_max); rate_min = max(rate_min, codec_stream->rate_min); @@ -452,80 +632,152 @@ static void soc_pcm_init_runtime_hw(struct snd_pcm_substream *substream) rates = snd_pcm_rate_mask_intersect(codec_stream->rates, rates); } + /* Verify both a valid CPU DAI and a valid CODEC DAI were found */ + if (!chan_min || !cpu_chan_min) + return -EINVAL; + /* * chan min/max cannot be enforced if there are multiple CODEC DAIs - * connected to a single CPU DAI, use CPU DAI's directly and let + * connected to CPU DAI(s), use CPU DAI's directly and let * channel allocation be fixed up later */ if (rtd->num_codecs > 1) { - chan_min = cpu_stream->channels_min; - chan_max = cpu_stream->channels_max; + chan_min = cpu_chan_min; + chan_max = cpu_chan_max; } - hw->channels_min = max(chan_min, cpu_stream->channels_min); - hw->channels_max = min(chan_max, cpu_stream->channels_max); - if (hw->formats) - hw->formats &= formats & cpu_stream->formats; - else - hw->formats = formats & cpu_stream->formats; - hw->rates = snd_pcm_rate_mask_intersect(rates, cpu_stream->rates); + /* finally find a intersection between CODECs and CPUs */ + hw->channels_min = max(chan_min, cpu_chan_min); + hw->channels_max = min(chan_max, cpu_chan_max); + hw->formats = formats; + hw->rates = snd_pcm_rate_mask_intersect(rates, cpu_rates); - snd_pcm_limit_hw_rates(runtime); + snd_pcm_hw_limit_rates(hw); - hw->rate_min = max(hw->rate_min, cpu_stream->rate_min); + hw->rate_min = max(hw->rate_min, cpu_rate_min); hw->rate_min = max(hw->rate_min, rate_min); - hw->rate_max = min_not_zero(hw->rate_max, cpu_stream->rate_max); + hw->rate_max = min_not_zero(hw->rate_max, cpu_rate_max); hw->rate_max = min_not_zero(hw->rate_max, rate_max); + + return 0; +} +EXPORT_SYMBOL_GPL(snd_soc_runtime_calc_hw); + +static void soc_pcm_init_runtime_hw(struct snd_pcm_substream *substream) +{ + struct snd_pcm_hardware *hw = &substream->runtime->hw; + struct snd_soc_pcm_runtime *rtd = substream->private_data; + u64 formats = hw->formats; + + /* + * At least one CPU and one CODEC should match. Otherwise, we should + * have bailed out on a higher level, since there would be no CPU or + * CODEC to support the transfer direction in that case. + */ + snd_soc_runtime_calc_hw(rtd, hw, substream->stream); + + if (formats) + hw->formats &= formats; } -static int soc_pcm_components_open(struct snd_pcm_substream *substream, - struct snd_soc_component **last) +static int soc_pcm_components_open(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_component *last = NULL; struct snd_soc_component *component; int i, ret = 0; for_each_rtd_components(rtd, i, component) { - *last = component; + last = component; ret = snd_soc_component_module_get_when_open(component); if (ret < 0) { dev_err(component->dev, "ASoC: can't get module %s\n", component->name); - return ret; + break; } ret = snd_soc_component_open(component, substream); if (ret < 0) { + snd_soc_component_module_put_when_close(component); dev_err(component->dev, "ASoC: can't open component %s: %d\n", component->name, ret); - return ret; + break; } } - *last = NULL; - return 0; + + if (ret < 0) { + /* rollback on error */ + for_each_rtd_components(rtd, i, component) { + if (component == last) + break; + + snd_soc_component_close(component, substream); + snd_soc_component_module_put_when_close(component); + } + } + + return ret; } -static int soc_pcm_components_close(struct snd_pcm_substream *substream, - struct snd_soc_component *last) +static int soc_pcm_components_close(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_component *component; - int i, ret = 0; + int i, r, ret = 0; for_each_rtd_components(rtd, i, component) { - if (component == last) - break; + r = snd_soc_component_close(component, substream); + if (r < 0) + ret = r; /* use last ret */ - ret |= snd_soc_component_close(component, substream); snd_soc_component_module_put_when_close(component); } return ret; } +/* + * Called by ALSA when a PCM substream is closed. Private data can be + * freed here. The cpu DAI, codec DAI, machine and components are also + * shutdown. + */ +static int soc_pcm_close(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_component *component; + struct snd_soc_dai *dai; + int i; + + mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass); + + snd_soc_runtime_deactivate(rtd, substream->stream); + + for_each_rtd_dais(rtd, i, dai) + snd_soc_dai_shutdown(dai, substream); + + soc_rtd_shutdown(rtd, substream); + + soc_pcm_components_close(substream); + + snd_soc_dapm_stream_stop(rtd, substream->stream); + + mutex_unlock(&rtd->card->pcm_mutex); + + for_each_rtd_components(rtd, i, component) { + pm_runtime_mark_last_busy(component->dev); + pm_runtime_put_autosuspend(component->dev); + } + + for_each_rtd_components(rtd, i, component) + if (!component->active) + pinctrl_pm_select_sleep_state(component->dev); + + return 0; +} + /* * Called by ALSA when a PCM substream is opened, the runtime->hw record is * then initialized and any private data can be allocated. This also calls @@ -536,9 +788,9 @@ static int soc_pcm_open(struct snd_pcm_substream *substream) struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_component *component; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; const char *codec_dai_name = "multicodec"; + const char *cpu_dai_name = "multicpu"; int i, ret = 0; for_each_rtd_components(rtd, i, component) @@ -549,38 +801,31 @@ static int soc_pcm_open(struct snd_pcm_substream *substream) mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass); - /* startup the audio subsystem */ - ret = snd_soc_dai_startup(cpu_dai, substream); - if (ret < 0) { - dev_err(cpu_dai->dev, "ASoC: can't open interface %s: %d\n", - cpu_dai->name, ret); - goto out; - } - - ret = soc_pcm_components_open(substream, &component); + ret = soc_pcm_components_open(substream); if (ret < 0) goto component_err; - for_each_rtd_codec_dai(rtd, i, codec_dai) { - ret = snd_soc_dai_startup(codec_dai, substream); - if (ret < 0) { - dev_err(codec_dai->dev, - "ASoC: can't open codec %s: %d\n", - codec_dai->name, ret); - goto codec_dai_err; - } - - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) - codec_dai->tx_mask = 0; - else - codec_dai->rx_mask = 0; - } - ret = soc_rtd_startup(rtd, substream); if (ret < 0) { pr_err("ASoC: %s startup failed: %d\n", rtd->dai_link->name, ret); - goto machine_err; + goto rtd_startup_err; + } + + /* startup the audio subsystem */ + for_each_rtd_dais(rtd, i, dai) { + ret = snd_soc_dai_startup(dai, substream); + if (ret < 0) { + dev_err(dai->dev, + "ASoC: can't open DAI %s: %d\n", + dai->name, ret); + goto config_err; + } + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + dai->tx_mask = 0; + else + dai->rx_mask = 0; } /* Dynamic PCM DAI links compat checks use dynamic capabilities */ @@ -593,46 +838,43 @@ static int soc_pcm_open(struct snd_pcm_substream *substream) if (rtd->num_codecs == 1) codec_dai_name = rtd->codec_dai->name; + if (rtd->num_cpus == 1) + cpu_dai_name = rtd->cpu_dai->name; + if (soc_pcm_has_symmetry(substream)) runtime->hw.info |= SNDRV_PCM_INFO_JOINT_DUPLEX; ret = -EINVAL; if (!runtime->hw.rates) { printk(KERN_ERR "ASoC: %s <-> %s No matching rates\n", - codec_dai_name, cpu_dai->name); + codec_dai_name, cpu_dai_name); goto config_err; } if (!runtime->hw.formats) { printk(KERN_ERR "ASoC: %s <-> %s No matching formats\n", - codec_dai_name, cpu_dai->name); + codec_dai_name, cpu_dai_name); goto config_err; } if (!runtime->hw.channels_min || !runtime->hw.channels_max || runtime->hw.channels_min > runtime->hw.channels_max) { printk(KERN_ERR "ASoC: %s <-> %s No matching channels\n", - codec_dai_name, cpu_dai->name); + codec_dai_name, cpu_dai_name); goto config_err; } soc_pcm_apply_msb(substream); /* Symmetry only applies if we've already got an active stream. */ - if (cpu_dai->active) { - ret = soc_pcm_apply_symmetry(substream, cpu_dai); - if (ret != 0) - goto config_err; - } - - for_each_rtd_codec_dai(rtd, i, codec_dai) { - if (codec_dai->active) { - ret = soc_pcm_apply_symmetry(substream, codec_dai); + for_each_rtd_dais(rtd, i, dai) { + if (dai->active) { + ret = soc_pcm_apply_symmetry(substream, dai); if (ret != 0) goto config_err; } } pr_debug("ASoC: %s <-> %s info:\n", - codec_dai_name, cpu_dai->name); + codec_dai_name, cpu_dai_name); pr_debug("ASoC: rate mask 0x%x\n", runtime->hw.rates); pr_debug("ASoC: min ch %d max ch %d\n", runtime->hw.channels_min, runtime->hw.channels_max); @@ -647,20 +889,13 @@ static int soc_pcm_open(struct snd_pcm_substream *substream) return 0; config_err: + for_each_rtd_dais(rtd, i, dai) + snd_soc_dai_shutdown(dai, substream); + soc_rtd_shutdown(rtd, substream); - -machine_err: - i = rtd->num_codecs; - -codec_dai_err: - for_each_rtd_codec_dai_rollback(rtd, i, codec_dai) - snd_soc_dai_shutdown(codec_dai, substream); - +rtd_startup_err: + soc_pcm_components_close(substream); component_err: - soc_pcm_components_close(substream, component); - - snd_soc_dai_shutdown(cpu_dai, substream); -out: mutex_unlock(&rtd->card->pcm_mutex); for_each_rtd_components(rtd, i, component) { @@ -685,59 +920,6 @@ static void codec2codec_close_delayed_work(struct snd_soc_pcm_runtime *rtd) */ } -/* - * Called by ALSA when a PCM substream is closed. Private data can be - * freed here. The cpu DAI, codec DAI, machine and components are also - * shutdown. - */ -static int soc_pcm_close(struct snd_pcm_substream *substream) -{ - struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_component *component; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; - int i; - - mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass); - - snd_soc_runtime_deactivate(rtd, substream->stream); - - /* clear the corresponding DAIs rate when inactive */ - if (!cpu_dai->active) - cpu_dai->rate = 0; - - for_each_rtd_codec_dai(rtd, i, codec_dai) { - if (!codec_dai->active) - codec_dai->rate = 0; - } - - snd_soc_dai_digital_mute(cpu_dai, 1, substream->stream); - - snd_soc_dai_shutdown(cpu_dai, substream); - - for_each_rtd_codec_dai(rtd, i, codec_dai) - snd_soc_dai_shutdown(codec_dai, substream); - - soc_rtd_shutdown(rtd, substream); - - soc_pcm_components_close(substream, NULL); - - snd_soc_dapm_stream_stop(rtd, substream->stream); - - mutex_unlock(&rtd->card->pcm_mutex); - - for_each_rtd_components(rtd, i, component) { - pm_runtime_mark_last_busy(component->dev); - pm_runtime_put_autosuspend(component->dev); - } - - for_each_rtd_components(rtd, i, component) - if (!component->active) - pinctrl_pm_select_sleep_state(component->dev); - - return 0; -} - /* * Called by ALSA when the PCM substream is prepared, can set format, sample * rate, etc. This function is non atomic and can be called multiple times, @@ -747,8 +929,7 @@ static int soc_pcm_prepare(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_component *component; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; int i, ret = 0; mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass); @@ -769,23 +950,15 @@ static int soc_pcm_prepare(struct snd_pcm_substream *substream) } } - for_each_rtd_codec_dai(rtd, i, codec_dai) { - ret = snd_soc_dai_prepare(codec_dai, substream); + for_each_rtd_dais(rtd, i, dai) { + ret = snd_soc_dai_prepare(dai, substream); if (ret < 0) { - dev_err(codec_dai->dev, - "ASoC: codec DAI prepare error: %d\n", - ret); + dev_err(dai->dev, + "ASoC: DAI prepare error: %d\n", ret); goto out; } } - ret = snd_soc_dai_prepare(cpu_dai, substream); - if (ret < 0) { - dev_err(cpu_dai->dev, - "ASoC: cpu DAI prepare error: %d\n", ret); - goto out; - } - /* cancel any delayed stream shutdown that is pending */ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && rtd->pop_wait) { @@ -796,10 +969,8 @@ static int soc_pcm_prepare(struct snd_pcm_substream *substream) snd_soc_dapm_stream_event(rtd, substream->stream, SND_SOC_DAPM_STREAM_START); - for_each_rtd_codec_dai(rtd, i, codec_dai) - snd_soc_dai_digital_mute(codec_dai, 0, - substream->stream); - snd_soc_dai_digital_mute(cpu_dai, 0, substream->stream); + for_each_rtd_dais(rtd, i, dai) + snd_soc_dai_digital_mute(dai, 0, substream->stream); out: mutex_unlock(&rtd->card->pcm_mutex); @@ -822,13 +993,15 @@ static int soc_pcm_components_hw_free(struct snd_pcm_substream *substream, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_component *component; - int i, ret = 0; + int i, r, ret = 0; for_each_rtd_components(rtd, i, component) { if (component == last) break; - ret |= snd_soc_component_hw_free(component, substream); + r = snd_soc_component_hw_free(component, substream); + if (r < 0) + ret = r; /* use last ret */ } return ret; @@ -844,7 +1017,7 @@ static int soc_pcm_hw_params(struct snd_pcm_substream *substream, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_component *component; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai; struct snd_soc_dai *codec_dai; int i, ret = 0; @@ -870,7 +1043,7 @@ static int soc_pcm_hw_params(struct snd_pcm_substream *substream, goto out; } - for_each_rtd_codec_dai(rtd, i, codec_dai) { + for_each_rtd_codec_dais(rtd, i, codec_dai) { struct snd_pcm_hw_params codec_params; /* @@ -917,17 +1090,26 @@ static int soc_pcm_hw_params(struct snd_pcm_substream *substream, snd_soc_dapm_update_dai(substream, &codec_params, codec_dai); } - ret = snd_soc_dai_hw_params(cpu_dai, substream, params); - if (ret < 0) - goto interface_err; + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { + /* + * Skip CPUs which don't support the current stream + * type. See soc_pcm_init_runtime_hw() for more details + */ + if (!snd_soc_dai_stream_valid(cpu_dai, substream->stream)) + continue; - /* store the parameters for each DAIs */ - cpu_dai->rate = params_rate(params); - cpu_dai->channels = params_channels(params); - cpu_dai->sample_bits = - snd_pcm_format_physical_width(params_format(params)); + ret = snd_soc_dai_hw_params(cpu_dai, substream, params); + if (ret < 0) + goto interface_err; - snd_soc_dapm_update_dai(substream, params, cpu_dai); + /* store the parameters for each DAI */ + cpu_dai->rate = params_rate(params); + cpu_dai->channels = params_channels(params); + cpu_dai->sample_bits = + snd_pcm_format_physical_width(params_format(params)); + + snd_soc_dapm_update_dai(substream, params, cpu_dai); + } for_each_rtd_components(rtd, i, component) { ret = snd_soc_component_hw_params(component, substream, params); @@ -947,14 +1129,21 @@ static int soc_pcm_hw_params(struct snd_pcm_substream *substream, component_err: soc_pcm_components_hw_free(substream, component); - snd_soc_dai_hw_free(cpu_dai, substream); - cpu_dai->rate = 0; + i = rtd->num_cpus; interface_err: + for_each_rtd_cpu_dais_rollback(rtd, i, cpu_dai) { + if (!snd_soc_dai_stream_valid(cpu_dai, substream->stream)) + continue; + + snd_soc_dai_hw_free(cpu_dai, substream); + cpu_dai->rate = 0; + } + i = rtd->num_codecs; codec_err: - for_each_rtd_codec_dai_rollback(rtd, i, codec_dai) { + for_each_rtd_codec_dais_rollback(rtd, i, codec_dai) { if (!snd_soc_dai_stream_valid(codec_dai, substream->stream)) continue; @@ -974,34 +1163,23 @@ static int soc_pcm_hw_params(struct snd_pcm_substream *substream, static int soc_pcm_hw_free(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; - bool playback = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; + struct snd_soc_dai *dai; int i; mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass); /* clear the corresponding DAIs parameters when going to be inactive */ - if (cpu_dai->active == 1) { - cpu_dai->rate = 0; - cpu_dai->channels = 0; - cpu_dai->sample_bits = 0; - } + for_each_rtd_dais(rtd, i, dai) { + int active = dai->stream_active[substream->stream]; - for_each_rtd_codec_dai(rtd, i, codec_dai) { - if (codec_dai->active == 1) { - codec_dai->rate = 0; - codec_dai->channels = 0; - codec_dai->sample_bits = 0; + if (dai->active == 1) { + dai->rate = 0; + dai->channels = 0; + dai->sample_bits = 0; } - } - /* apply codec digital mute */ - for_each_rtd_codec_dai(rtd, i, codec_dai) { - if ((playback && codec_dai->playback_active == 1) || - (!playback && codec_dai->capture_active == 1)) - snd_soc_dai_digital_mute(codec_dai, 1, - substream->stream); + if (active == 1) + snd_soc_dai_digital_mute(dai, 1, substream->stream); } /* free any machine hw params */ @@ -1011,15 +1189,13 @@ static int soc_pcm_hw_free(struct snd_pcm_substream *substream) soc_pcm_components_hw_free(substream, NULL); /* now free hw params for the DAIs */ - for_each_rtd_codec_dai(rtd, i, codec_dai) { - if (!snd_soc_dai_stream_valid(codec_dai, substream->stream)) + for_each_rtd_dais(rtd, i, dai) { + if (!snd_soc_dai_stream_valid(dai, substream->stream)) continue; - snd_soc_dai_hw_free(codec_dai, substream); + snd_soc_dai_hw_free(dai, substream); } - snd_soc_dai_hw_free(cpu_dai, substream); - mutex_unlock(&rtd->card->pcm_mutex); return 0; } @@ -1028,8 +1204,7 @@ static int soc_pcm_trigger_start(struct snd_pcm_substream *substream, int cmd) { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_component *component; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; int i, ret; ret = soc_rtd_trigger(rtd, substream, cmd); @@ -1042,12 +1217,8 @@ static int soc_pcm_trigger_start(struct snd_pcm_substream *substream, int cmd) return ret; } - ret = snd_soc_dai_trigger(cpu_dai, substream, cmd); - if (ret < 0) - return ret; - - for_each_rtd_codec_dai(rtd, i, codec_dai) { - ret = snd_soc_dai_trigger(codec_dai, substream, cmd); + for_each_rtd_dais(rtd, i, dai) { + ret = snd_soc_dai_trigger(dai, substream, cmd); if (ret < 0) return ret; } @@ -1059,20 +1230,15 @@ static int soc_pcm_trigger_stop(struct snd_pcm_substream *substream, int cmd) { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_component *component; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; int i, ret; - for_each_rtd_codec_dai(rtd, i, codec_dai) { - ret = snd_soc_dai_trigger(codec_dai, substream, cmd); + for_each_rtd_dais(rtd, i, dai) { + ret = snd_soc_dai_trigger(dai, substream, cmd); if (ret < 0) return ret; } - ret = snd_soc_dai_trigger(cpu_dai, substream, cmd); - if (ret < 0) - return ret; - for_each_rtd_components(rtd, i, component) { ret = snd_soc_component_trigger(component, substream, cmd); if (ret < 0) @@ -1112,20 +1278,15 @@ static int soc_pcm_bespoke_trigger(struct snd_pcm_substream *substream, int cmd) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; int i, ret; - for_each_rtd_codec_dai(rtd, i, codec_dai) { - ret = snd_soc_dai_bespoke_trigger(codec_dai, substream, cmd); + for_each_rtd_dais(rtd, i, dai) { + ret = snd_soc_dai_bespoke_trigger(dai, substream, cmd); if (ret < 0) return ret; } - ret = snd_soc_dai_bespoke_trigger(cpu_dai, substream, cmd); - if (ret < 0) - return ret; - return 0; } /* @@ -1136,12 +1297,13 @@ static int soc_pcm_bespoke_trigger(struct snd_pcm_substream *substream, static snd_pcm_uframes_t soc_pcm_pointer(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai; struct snd_soc_dai *codec_dai; struct snd_pcm_runtime *runtime = substream->runtime; snd_pcm_uframes_t offset = 0; snd_pcm_sframes_t delay = 0; snd_pcm_sframes_t codec_delay = 0; + snd_pcm_sframes_t cpu_delay = 0; int i; /* clearing the previous total delay */ @@ -1152,9 +1314,13 @@ static snd_pcm_uframes_t soc_pcm_pointer(struct snd_pcm_substream *substream) /* base delay if assigned in pointer callback */ delay = runtime->delay; - delay += snd_soc_dai_delay(cpu_dai, substream); + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { + cpu_delay = max(cpu_delay, + snd_soc_dai_delay(cpu_dai, substream)); + } + delay += cpu_delay; - for_each_rtd_codec_dai(rtd, i, codec_dai) { + for_each_rtd_codec_dais(rtd, i, codec_dai) { codec_delay = max(codec_delay, snd_soc_dai_delay(codec_dai, substream)); } @@ -1171,9 +1337,6 @@ static int dpcm_be_connect(struct snd_soc_pcm_runtime *fe, { struct snd_soc_dpcm *dpcm; unsigned long flags; -#ifdef CONFIG_DEBUG_FS - char *name; -#endif /* only add new dpcms */ for_each_dpcm_be(fe, stream, dpcm) { @@ -1198,17 +1361,8 @@ static int dpcm_be_connect(struct snd_soc_pcm_runtime *fe, stream ? "capture" : "playback", fe->dai_link->name, stream ? "<-" : "->", be->dai_link->name); -#ifdef CONFIG_DEBUG_FS - name = kasprintf(GFP_KERNEL, "%s:%s", be->dai_link->name, - stream ? "capture" : "playback"); - if (name) { - dpcm->debugfs_state = debugfs_create_dir(name, - fe->debugfs_dpcm_root); - debugfs_create_u32("state", 0644, dpcm->debugfs_state, - &dpcm->state); - kfree(name); - } -#endif + dpcm_create_debugfs_state(dpcm, stream); + return 1; } @@ -1261,9 +1415,8 @@ void dpcm_be_disconnect(struct snd_soc_pcm_runtime *fe, int stream) /* BEs still alive need new FE */ dpcm_be_reparent(fe, dpcm->be, stream); -#ifdef CONFIG_DEBUG_FS - debugfs_remove_recursive(dpcm->debugfs_state); -#endif + dpcm_remove_debugfs_state(dpcm); + spin_lock_irqsave(&fe->card->dpcm_lock, flags); list_del(&dpcm->list_be); list_del(&dpcm->list_fe); @@ -1277,74 +1430,41 @@ static struct snd_soc_pcm_runtime *dpcm_get_be(struct snd_soc_card *card, struct snd_soc_dapm_widget *widget, int stream) { struct snd_soc_pcm_runtime *be; + struct snd_soc_dapm_widget *w; struct snd_soc_dai *dai; int i; dev_dbg(card->dev, "ASoC: find BE for widget %s\n", widget->name); - if (stream == SNDRV_PCM_STREAM_PLAYBACK) { - for_each_card_rtds(card, be) { + for_each_card_rtds(card, be) { - if (!be->dai_link->no_pcm) - continue; + if (!be->dai_link->no_pcm) + continue; + + for_each_rtd_dais(be, i, dai) { + w = snd_soc_dai_get_widget(dai, stream); dev_dbg(card->dev, "ASoC: try BE : %s\n", - be->cpu_dai->playback_widget ? - be->cpu_dai->playback_widget->name : "(not set)"); + w ? w->name : "(not set)"); - if (be->cpu_dai->playback_widget == widget) + if (w == widget) return be; - - for_each_rtd_codec_dai(be, i, dai) { - if (dai->playback_widget == widget) - return be; - } - } - } else { - - for_each_card_rtds(card, be) { - - if (!be->dai_link->no_pcm) - continue; - - dev_dbg(card->dev, "ASoC: try BE %s\n", - be->cpu_dai->capture_widget ? - be->cpu_dai->capture_widget->name : "(not set)"); - - if (be->cpu_dai->capture_widget == widget) - return be; - - for_each_rtd_codec_dai(be, i, dai) { - if (dai->capture_widget == widget) - return be; - } } } - /* dai link name and stream name set correctly ? */ - dev_err(card->dev, "ASoC: can't get %s BE for %s\n", - stream ? "capture" : "playback", widget->name); + /* Widget provided is not a BE */ return NULL; } -static inline struct snd_soc_dapm_widget * - dai_get_widget(struct snd_soc_dai *dai, int stream) -{ - if (stream == SNDRV_PCM_STREAM_PLAYBACK) - return dai->playback_widget; - else - return dai->capture_widget; -} - static int widget_in_list(struct snd_soc_dapm_widget_list *list, struct snd_soc_dapm_widget *widget) { + struct snd_soc_dapm_widget *w; int i; - for (i = 0; i < list->num_widgets; i++) { - if (widget == list->widgets[i]) + for_each_dapm_widgets(list, i, w) + if (widget == w) return 1; - } return 0; } @@ -1354,36 +1474,17 @@ static bool dpcm_end_walk_at_be(struct snd_soc_dapm_widget *widget, { struct snd_soc_card *card = widget->dapm->card; struct snd_soc_pcm_runtime *rtd; - struct snd_soc_dai *dai; - int i; + int stream; - if (dir == SND_SOC_DAPM_DIR_OUT) { - for_each_card_rtds(card, rtd) { - if (!rtd->dai_link->no_pcm) - continue; + /* adjust dir to stream */ + if (dir == SND_SOC_DAPM_DIR_OUT) + stream = SNDRV_PCM_STREAM_PLAYBACK; + else + stream = SNDRV_PCM_STREAM_CAPTURE; - if (rtd->cpu_dai->playback_widget == widget) - return true; - - for_each_rtd_codec_dai(rtd, i, dai) { - if (dai->playback_widget == widget) - return true; - } - } - } else { /* SND_SOC_DAPM_DIR_IN */ - for_each_card_rtds(card, rtd) { - if (!rtd->dai_link->no_pcm) - continue; - - if (rtd->cpu_dai->capture_widget == widget) - return true; - - for_each_rtd_codec_dai(rtd, i, dai) { - if (dai->capture_widget == widget) - return true; - } - } - } + rtd = dpcm_get_be(card, widget, stream); + if (rtd) + return true; return false; } @@ -1394,6 +1495,12 @@ int dpcm_path_get(struct snd_soc_pcm_runtime *fe, struct snd_soc_dai *cpu_dai = fe->cpu_dai; int paths; + if (fe->num_cpus > 1) { + dev_err(fe->dev, + "%s doesn't support Multi CPU yet\n", __func__); + return -EINVAL; + } + /* get number of valid DAI paths and their widgets */ paths = snd_soc_dapm_dai_get_connected_widgets(cpu_dai, stream, list, dpcm_end_walk_at_be); @@ -1404,37 +1511,42 @@ int dpcm_path_get(struct snd_soc_pcm_runtime *fe, return paths; } -static int dpcm_prune_paths(struct snd_soc_pcm_runtime *fe, int stream, - struct snd_soc_dapm_widget_list **list_) +void dpcm_path_put(struct snd_soc_dapm_widget_list **list) +{ + snd_soc_dapm_dai_free_widgets(list); +} + +static bool dpcm_be_is_active(struct snd_soc_dpcm *dpcm, int stream, + struct snd_soc_dapm_widget_list *list) { - struct snd_soc_dpcm *dpcm; - struct snd_soc_dapm_widget_list *list = *list_; struct snd_soc_dapm_widget *widget; struct snd_soc_dai *dai; + unsigned int i; + + /* is there a valid DAI widget for this BE */ + for_each_rtd_dais(dpcm->be, i, dai) { + widget = snd_soc_dai_get_widget(dai, stream); + + /* + * The BE is pruned only if none of the dai + * widgets are in the active list. + */ + if (widget && widget_in_list(list, widget)) + return true; + } + + return false; +} + +static int dpcm_prune_paths(struct snd_soc_pcm_runtime *fe, int stream, + struct snd_soc_dapm_widget_list **list_) +{ + struct snd_soc_dpcm *dpcm; int prune = 0; - int do_prune; /* Destroy any old FE <--> BE connections */ for_each_dpcm_be(fe, stream, dpcm) { - unsigned int i; - - /* is there a valid CPU DAI widget for this BE */ - widget = dai_get_widget(dpcm->be->cpu_dai, stream); - - /* prune the BE if it's no longer in our active list */ - if (widget && widget_in_list(list, widget)) - continue; - - /* is there a valid CODEC DAI widget for this BE */ - do_prune = 1; - for_each_rtd_codec_dai(dpcm->be, i, dai) { - widget = dai_get_widget(dai, stream); - - /* prune the BE if it's no longer in our active list */ - if (widget && widget_in_list(list, widget)) - do_prune = 0; - } - if (!do_prune) + if (dpcm_be_is_active(dpcm, stream, *list_)) continue; dev_dbg(fe->dev, "ASoC: pruning %s BE %s for %s\n", @@ -1455,12 +1567,13 @@ static int dpcm_add_paths(struct snd_soc_pcm_runtime *fe, int stream, struct snd_soc_card *card = fe->card; struct snd_soc_dapm_widget_list *list = *list_; struct snd_soc_pcm_runtime *be; + struct snd_soc_dapm_widget *widget; int i, new = 0, err; /* Create any new FE <--> BE connections */ - for (i = 0; i < list->num_widgets; i++) { + for_each_dapm_widgets(list, i, widget) { - switch (list->widgets[i]->id) { + switch (widget->id) { case snd_soc_dapm_dai_in: if (stream != SNDRV_PCM_STREAM_PLAYBACK) continue; @@ -1474,17 +1587,13 @@ static int dpcm_add_paths(struct snd_soc_pcm_runtime *fe, int stream, } /* is there a valid BE rtd for this widget */ - be = dpcm_get_be(card, list->widgets[i], stream); + be = dpcm_get_be(card, widget, stream); if (!be) { dev_err(fe->dev, "ASoC: no BE found for %s\n", - list->widgets[i]->name); + widget->name); continue; } - /* make sure BE is a real BE */ - if (!be->dai_link->no_pcm) - continue; - /* don't connect if FE is not running */ if (!fe->dpcm[stream].runtime && !fe->fe_compr) continue; @@ -1493,7 +1602,7 @@ static int dpcm_add_paths(struct snd_soc_pcm_runtime *fe, int stream, err = dpcm_be_connect(fe, be, stream); if (err < 0) { dev_err(fe->dev, "ASoC: can't connect %s\n", - list->widgets[i]->name); + widget->name); break; } else if (err == 0) /* already connected */ continue; @@ -1680,11 +1789,10 @@ static void dpcm_runtime_merge_format(struct snd_pcm_substream *substream, for_each_dpcm_be(fe, stream, dpcm) { struct snd_soc_pcm_runtime *be = dpcm->be; - struct snd_soc_dai_driver *codec_dai_drv; struct snd_soc_pcm_stream *codec_stream; int i; - for_each_rtd_codec_dai(be, i, dai) { + for_each_rtd_codec_dais(be, i, dai) { /* * Skip CODECs which don't support the current stream * type. See soc_pcm_init_runtime_hw() for more details @@ -1692,11 +1800,7 @@ static void dpcm_runtime_merge_format(struct snd_pcm_substream *substream, if (!snd_soc_dai_stream_valid(dai, stream)) continue; - codec_dai_drv = dai->driver; - if (stream == SNDRV_PCM_STREAM_PLAYBACK) - codec_stream = &codec_dai_drv->playback; - else - codec_stream = &codec_dai_drv->capture; + codec_stream = snd_soc_dai_get_pcm_stream(dai, stream); *formats &= codec_stream->formats; } @@ -1721,30 +1825,33 @@ static void dpcm_runtime_merge_chan(struct snd_pcm_substream *substream, for_each_dpcm_be(fe, stream, dpcm) { struct snd_soc_pcm_runtime *be = dpcm->be; - struct snd_soc_dai_driver *cpu_dai_drv = be->cpu_dai->driver; - struct snd_soc_dai_driver *codec_dai_drv; struct snd_soc_pcm_stream *codec_stream; struct snd_soc_pcm_stream *cpu_stream; + struct snd_soc_dai *dai; + int i; - if (stream == SNDRV_PCM_STREAM_PLAYBACK) - cpu_stream = &cpu_dai_drv->playback; - else - cpu_stream = &cpu_dai_drv->capture; + for_each_rtd_cpu_dais(be, i, dai) { + /* + * Skip CPUs which don't support the current stream + * type. See soc_pcm_init_runtime_hw() for more details + */ + if (!snd_soc_dai_stream_valid(dai, stream)) + continue; - *channels_min = max(*channels_min, cpu_stream->channels_min); - *channels_max = min(*channels_max, cpu_stream->channels_max); + cpu_stream = snd_soc_dai_get_pcm_stream(dai, stream); + + *channels_min = max(*channels_min, + cpu_stream->channels_min); + *channels_max = min(*channels_max, + cpu_stream->channels_max); + } /* * chan min/max cannot be enforced if there are multiple CODEC * DAIs connected to a single CPU DAI, use CPU DAI's directly */ if (be->num_codecs == 1) { - codec_dai_drv = be->codec_dais[0]->driver; - - if (stream == SNDRV_PCM_STREAM_PLAYBACK) - codec_stream = &codec_dai_drv->playback; - else - codec_stream = &codec_dai_drv->capture; + codec_stream = snd_soc_dai_get_pcm_stream(be->codec_dais[0], stream); *channels_min = max(*channels_min, codec_stream->channels_min); @@ -1773,41 +1880,23 @@ static void dpcm_runtime_merge_rate(struct snd_pcm_substream *substream, for_each_dpcm_be(fe, stream, dpcm) { struct snd_soc_pcm_runtime *be = dpcm->be; - struct snd_soc_dai_driver *cpu_dai_drv = be->cpu_dai->driver; - struct snd_soc_dai_driver *codec_dai_drv; - struct snd_soc_pcm_stream *codec_stream; - struct snd_soc_pcm_stream *cpu_stream; + struct snd_soc_pcm_stream *pcm; struct snd_soc_dai *dai; int i; - if (stream == SNDRV_PCM_STREAM_PLAYBACK) - cpu_stream = &cpu_dai_drv->playback; - else - cpu_stream = &cpu_dai_drv->capture; - - *rate_min = max(*rate_min, cpu_stream->rate_min); - *rate_max = min_not_zero(*rate_max, cpu_stream->rate_max); - *rates = snd_pcm_rate_mask_intersect(*rates, cpu_stream->rates); - - for_each_rtd_codec_dai(be, i, dai) { + for_each_rtd_dais(be, i, dai) { /* - * Skip CODECs which don't support the current stream + * Skip DAIs which don't support the current stream * type. See soc_pcm_init_runtime_hw() for more details */ if (!snd_soc_dai_stream_valid(dai, stream)) continue; - codec_dai_drv = dai->driver; - if (stream == SNDRV_PCM_STREAM_PLAYBACK) - codec_stream = &codec_dai_drv->playback; - else - codec_stream = &codec_dai_drv->capture; + pcm = snd_soc_dai_get_pcm_stream(dai, stream); - *rate_min = max(*rate_min, codec_stream->rate_min); - *rate_max = min_not_zero(*rate_max, - codec_stream->rate_max); - *rates = snd_pcm_rate_mask_intersect(*rates, - codec_stream->rates); + *rate_min = max(*rate_min, pcm->rate_min); + *rate_max = min_not_zero(*rate_max, pcm->rate_max); + *rates = snd_pcm_rate_mask_intersect(*rates, pcm->rates); } } } @@ -1816,13 +1905,21 @@ static void dpcm_set_fe_runtime(struct snd_pcm_substream *substream) { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; - struct snd_soc_dai_driver *cpu_dai_drv = cpu_dai->driver; + struct snd_soc_dai *cpu_dai; + int i; - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) - dpcm_init_runtime_hw(runtime, &cpu_dai_drv->playback); - else - dpcm_init_runtime_hw(runtime, &cpu_dai_drv->capture); + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { + /* + * Skip CPUs which don't support the current stream + * type. See soc_pcm_init_runtime_hw() for more details + */ + if (!snd_soc_dai_stream_valid(cpu_dai, substream->stream)) + continue; + + dpcm_init_runtime_hw(runtime, + snd_soc_dai_get_pcm_stream(cpu_dai, + substream->stream)); + } dpcm_runtime_merge_format(substream, &runtime->hw.formats); dpcm_runtime_merge_chan(substream, &runtime->hw.channels_min, @@ -1859,18 +1956,21 @@ static int dpcm_apply_symmetry(struct snd_pcm_substream *fe_substream, { struct snd_soc_dpcm *dpcm; struct snd_soc_pcm_runtime *fe = fe_substream->private_data; - struct snd_soc_dai *fe_cpu_dai = fe->cpu_dai; + struct snd_soc_dai *fe_cpu_dai; int err; + int i; /* apply symmetry for FE */ if (soc_pcm_has_symmetry(fe_substream)) fe_substream->runtime->hw.info |= SNDRV_PCM_INFO_JOINT_DUPLEX; - /* Symmetry only applies if we've got an active stream. */ - if (fe_cpu_dai->active) { - err = soc_pcm_apply_symmetry(fe_substream, fe_cpu_dai); - if (err < 0) - return err; + for_each_rtd_cpu_dais (fe, i, fe_cpu_dai) { + /* Symmetry only applies if we've got an active stream. */ + if (fe_cpu_dai->active) { + err = soc_pcm_apply_symmetry(fe_substream, fe_cpu_dai); + if (err < 0) + return err; + } } /* apply symmetry for BE */ @@ -1879,7 +1979,7 @@ static int dpcm_apply_symmetry(struct snd_pcm_substream *fe_substream, struct snd_pcm_substream *be_substream = snd_soc_dpcm_get_substream(be, stream); struct snd_soc_pcm_runtime *rtd; - struct snd_soc_dai *codec_dai; + struct snd_soc_dai *dai; int i; /* A backend may not have the requested substream */ @@ -1894,17 +1994,9 @@ static int dpcm_apply_symmetry(struct snd_pcm_substream *fe_substream, be_substream->runtime->hw.info |= SNDRV_PCM_INFO_JOINT_DUPLEX; /* Symmetry only applies if we've got an active stream. */ - if (rtd->cpu_dai->active) { - err = soc_pcm_apply_symmetry(fe_substream, - rtd->cpu_dai); - if (err < 0) - return err; - } - - for_each_rtd_codec_dai(rtd, i, codec_dai) { - if (codec_dai->active) { - err = soc_pcm_apply_symmetry(fe_substream, - codec_dai); + for_each_rtd_dais(rtd, i, dai) { + if (dai->active) { + err = soc_pcm_apply_symmetry(fe_substream, dai); if (err < 0) return err; } @@ -1922,7 +2014,7 @@ static int dpcm_fe_dai_startup(struct snd_pcm_substream *fe_substream) dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_FE); - ret = dpcm_be_dai_startup(fe, fe_substream->stream); + ret = dpcm_be_dai_startup(fe, stream); if (ret < 0) { dev_err(fe->dev,"ASoC: failed to start some BEs %d\n", ret); goto be_err; @@ -1943,17 +2035,13 @@ static int dpcm_fe_dai_startup(struct snd_pcm_substream *fe_substream) snd_pcm_limit_hw_rates(runtime); ret = dpcm_apply_symmetry(fe_substream, stream); - if (ret < 0) { + if (ret < 0) dev_err(fe->dev, "ASoC: failed to apply dpcm symmetry %d\n", ret); - goto unwind; - } - - dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO); - return 0; unwind: - dpcm_be_dai_startup_unwind(fe, fe_substream->stream); + if (ret < 0) + dpcm_be_dai_startup_unwind(fe, stream); be_err: dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO); return ret; @@ -2007,7 +2095,7 @@ static int dpcm_fe_dai_shutdown(struct snd_pcm_substream *substream) dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_FE); /* shutdown the BEs */ - dpcm_be_dai_shutdown(fe, substream->stream); + dpcm_be_dai_shutdown(fe, stream); dev_dbg(fe->dev, "ASoC: close FE %s\n", fe->dai_link->name); @@ -2185,9 +2273,9 @@ static int dpcm_fe_dai_hw_params(struct snd_pcm_substream *substream, mutex_lock_nested(&fe->card->mutex, SND_SOC_CARD_CLASS_RUNTIME); dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_FE); - memcpy(&fe->dpcm[substream->stream].hw_params, params, + memcpy(&fe->dpcm[stream].hw_params, params, sizeof(struct snd_pcm_hw_params)); - ret = dpcm_be_dai_hw_params(fe, substream->stream); + ret = dpcm_be_dai_hw_params(fe, stream); if (ret < 0) { dev_err(fe->dev,"ASoC: hw_params BE failed %d\n", ret); goto out; @@ -2509,7 +2597,7 @@ static int dpcm_fe_dai_prepare(struct snd_pcm_substream *substream) goto out; } - ret = dpcm_be_dai_prepare(fe, substream->stream); + ret = dpcm_be_dai_prepare(fe, stream); if (ret < 0) goto out; @@ -2661,36 +2749,18 @@ static int dpcm_run_update_startup(struct snd_soc_pcm_runtime *fe, int stream) return ret; } -static int dpcm_run_new_update(struct snd_soc_pcm_runtime *fe, int stream) -{ - int ret; - - dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_BE); - ret = dpcm_run_update_startup(fe, stream); - if (ret < 0) - dev_err(fe->dev, "ASoC: failed to startup some BEs\n"); - dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO); - - return ret; -} - -static int dpcm_run_old_update(struct snd_soc_pcm_runtime *fe, int stream) -{ - int ret; - - dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_BE); - ret = dpcm_run_update_shutdown(fe, stream); - if (ret < 0) - dev_err(fe->dev, "ASoC: failed to shutdown some BEs\n"); - dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO); - - return ret; -} - static int soc_dpcm_fe_runtime_update(struct snd_soc_pcm_runtime *fe, int new) { struct snd_soc_dapm_widget_list *list; + int stream; int count, paths; + int ret; + + if (fe->num_cpus > 1) { + dev_err(fe->dev, + "%s doesn't support Multi CPU yet\n", __func__); + return -EINVAL; + } if (!fe->dai_link->dynamic) return 0; @@ -2703,74 +2773,53 @@ static int soc_dpcm_fe_runtime_update(struct snd_soc_pcm_runtime *fe, int new) dev_dbg(fe->dev, "ASoC: DPCM %s runtime update for FE %s\n", new ? "new" : "old", fe->dai_link->name); - /* skip if FE doesn't have playback capability */ - if (!snd_soc_dai_stream_valid(fe->cpu_dai, SNDRV_PCM_STREAM_PLAYBACK) || - !snd_soc_dai_stream_valid(fe->codec_dai, SNDRV_PCM_STREAM_PLAYBACK)) - goto capture; + for_each_pcm_streams(stream) { - /* skip if FE isn't currently playing */ - if (!fe->cpu_dai->playback_active || !fe->codec_dai->playback_active) - goto capture; + /* skip if FE doesn't have playback/capture capability */ + if (!snd_soc_dai_stream_valid(fe->cpu_dai, stream) || + !snd_soc_dai_stream_valid(fe->codec_dai, stream)) + continue; - paths = dpcm_path_get(fe, SNDRV_PCM_STREAM_PLAYBACK, &list); - if (paths < 0) { - dev_warn(fe->dev, "ASoC: %s no valid %s path\n", - fe->dai_link->name, "playback"); - return paths; + /* skip if FE isn't currently playing/capturing */ + if (!fe->cpu_dai->stream_active[stream] || + !fe->codec_dai->stream_active[stream]) + continue; + + paths = dpcm_path_get(fe, stream, &list); + if (paths < 0) { + dev_warn(fe->dev, "ASoC: %s no valid %s path\n", + fe->dai_link->name, + stream == SNDRV_PCM_STREAM_PLAYBACK ? + "playback" : "capture"); + return paths; + } + + /* update any playback/capture paths */ + count = dpcm_process_paths(fe, stream, &list, new); + if (count) { + dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_BE); + if (new) + ret = dpcm_run_update_startup(fe, stream); + else + ret = dpcm_run_update_shutdown(fe, stream); + if (ret < 0) + dev_err(fe->dev, "ASoC: failed to shutdown some BEs\n"); + dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO); + + dpcm_clear_pending_state(fe, stream); + dpcm_be_disconnect(fe, stream); + } + + dpcm_path_put(&list); } - /* update any playback paths */ - count = dpcm_process_paths(fe, SNDRV_PCM_STREAM_PLAYBACK, &list, new); - if (count) { - if (new) - dpcm_run_new_update(fe, SNDRV_PCM_STREAM_PLAYBACK); - else - dpcm_run_old_update(fe, SNDRV_PCM_STREAM_PLAYBACK); - - dpcm_clear_pending_state(fe, SNDRV_PCM_STREAM_PLAYBACK); - dpcm_be_disconnect(fe, SNDRV_PCM_STREAM_PLAYBACK); - } - - dpcm_path_put(&list); - -capture: - /* skip if FE doesn't have capture capability */ - if (!snd_soc_dai_stream_valid(fe->cpu_dai, SNDRV_PCM_STREAM_CAPTURE) || - !snd_soc_dai_stream_valid(fe->codec_dai, SNDRV_PCM_STREAM_CAPTURE)) - return 0; - - /* skip if FE isn't currently capturing */ - if (!fe->cpu_dai->capture_active || !fe->codec_dai->capture_active) - return 0; - - paths = dpcm_path_get(fe, SNDRV_PCM_STREAM_CAPTURE, &list); - if (paths < 0) { - dev_warn(fe->dev, "ASoC: %s no valid %s path\n", - fe->dai_link->name, "capture"); - return paths; - } - - /* update any old capture paths */ - count = dpcm_process_paths(fe, SNDRV_PCM_STREAM_CAPTURE, &list, new); - if (count) { - if (new) - dpcm_run_new_update(fe, SNDRV_PCM_STREAM_CAPTURE); - else - dpcm_run_old_update(fe, SNDRV_PCM_STREAM_CAPTURE); - - dpcm_clear_pending_state(fe, SNDRV_PCM_STREAM_CAPTURE); - dpcm_be_disconnect(fe, SNDRV_PCM_STREAM_CAPTURE); - } - - dpcm_path_put(&list); - return 0; } /* Called by DAPM mixer/mux changes to update audio routing between PCMs and * any DAI links. */ -int soc_dpcm_runtime_update(struct snd_soc_card *card) +int snd_soc_dpcm_runtime_update(struct snd_soc_card *card) { struct snd_soc_pcm_runtime *fe; int ret = 0; @@ -2794,38 +2843,40 @@ int soc_dpcm_runtime_update(struct snd_soc_card *card) mutex_unlock(&card->mutex); return ret; } -int soc_dpcm_be_digital_mute(struct snd_soc_pcm_runtime *fe, int mute) +EXPORT_SYMBOL_GPL(snd_soc_dpcm_runtime_update); + +static void dpcm_fe_dai_cleanup(struct snd_pcm_substream *fe_substream) { + struct snd_soc_pcm_runtime *fe = fe_substream->private_data; struct snd_soc_dpcm *dpcm; - struct snd_soc_dai *dai; + int stream = fe_substream->stream; - for_each_dpcm_be(fe, SNDRV_PCM_STREAM_PLAYBACK, dpcm) { + /* mark FE's links ready to prune */ + for_each_dpcm_be(fe, stream, dpcm) + dpcm->state = SND_SOC_DPCM_LINK_STATE_FREE; - struct snd_soc_pcm_runtime *be = dpcm->be; - int i; + dpcm_be_disconnect(fe, stream); - if (be->dai_link->ignore_suspend) - continue; + fe->dpcm[stream].runtime = NULL; +} - for_each_rtd_codec_dai(be, i, dai) { - struct snd_soc_dai_driver *drv = dai->driver; +static int dpcm_fe_dai_close(struct snd_pcm_substream *fe_substream) +{ + struct snd_soc_pcm_runtime *fe = fe_substream->private_data; + int ret; - dev_dbg(be->dev, "ASoC: BE digital mute %s\n", - be->dai_link->name); + mutex_lock_nested(&fe->card->mutex, SND_SOC_CARD_CLASS_RUNTIME); + ret = dpcm_fe_dai_shutdown(fe_substream); - if (drv->ops && drv->ops->digital_mute && - dai->playback_active) - drv->ops->digital_mute(dai, mute); - } - } + dpcm_fe_dai_cleanup(fe_substream); - return 0; + mutex_unlock(&fe->card->mutex); + return ret; } static int dpcm_fe_dai_open(struct snd_pcm_substream *fe_substream) { struct snd_soc_pcm_runtime *fe = fe_substream->private_data; - struct snd_soc_dpcm *dpcm; struct snd_soc_dapm_widget_list *list; int ret; int stream = fe_substream->stream; @@ -2835,8 +2886,7 @@ static int dpcm_fe_dai_open(struct snd_pcm_substream *fe_substream) ret = dpcm_path_get(fe, stream, &list); if (ret < 0) { - mutex_unlock(&fe->card->mutex); - return ret; + goto open_end; } else if (ret == 0) { dev_dbg(fe->dev, "ASoC: %s no valid %s route\n", fe->dai_link->name, stream ? "capture" : "playback"); @@ -2846,37 +2896,12 @@ static int dpcm_fe_dai_open(struct snd_pcm_substream *fe_substream) dpcm_process_paths(fe, stream, &list, 1); ret = dpcm_fe_dai_startup(fe_substream); - if (ret < 0) { - /* clean up all links */ - for_each_dpcm_be(fe, stream, dpcm) - dpcm->state = SND_SOC_DPCM_LINK_STATE_FREE; - - dpcm_be_disconnect(fe, stream); - fe->dpcm[stream].runtime = NULL; - } + if (ret < 0) + dpcm_fe_dai_cleanup(fe_substream); dpcm_clear_pending_state(fe, stream); dpcm_path_put(&list); - mutex_unlock(&fe->card->mutex); - return ret; -} - -static int dpcm_fe_dai_close(struct snd_pcm_substream *fe_substream) -{ - struct snd_soc_pcm_runtime *fe = fe_substream->private_data; - struct snd_soc_dpcm *dpcm; - int stream = fe_substream->stream, ret; - - mutex_lock_nested(&fe->card->mutex, SND_SOC_CARD_CLASS_RUNTIME); - ret = dpcm_fe_dai_shutdown(fe_substream); - - /* mark FE's links ready to prune */ - for_each_dpcm_be(fe, stream, dpcm) - dpcm->state = SND_SOC_DPCM_LINK_STATE_FREE; - - dpcm_be_disconnect(fe, stream); - - fe->dpcm[stream].runtime = NULL; +open_end: mutex_unlock(&fe->card->mutex); return ret; } @@ -2885,7 +2910,7 @@ static int dpcm_fe_dai_close(struct snd_pcm_substream *fe_substream) int soc_new_pcm(struct snd_soc_pcm_runtime *rtd, int num) { struct snd_soc_dai *codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai; struct snd_soc_component *component; struct snd_pcm *pcm; char new_name[64]; @@ -2897,22 +2922,29 @@ int soc_new_pcm(struct snd_soc_pcm_runtime *rtd, int num) capture = rtd->dai_link->dpcm_capture; } else { /* Adapt stream for codec2codec links */ - struct snd_soc_pcm_stream *cpu_capture = rtd->dai_link->params ? - &cpu_dai->driver->playback : &cpu_dai->driver->capture; - struct snd_soc_pcm_stream *cpu_playback = rtd->dai_link->params ? - &cpu_dai->driver->capture : &cpu_dai->driver->playback; + int cpu_capture = rtd->dai_link->params ? + SNDRV_PCM_STREAM_PLAYBACK : SNDRV_PCM_STREAM_CAPTURE; + int cpu_playback = rtd->dai_link->params ? + SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK; + + for_each_rtd_codec_dais(rtd, i, codec_dai) { + if (rtd->num_cpus == 1) { + cpu_dai = rtd->cpu_dais[0]; + } else if (rtd->num_cpus == rtd->num_codecs) { + cpu_dai = rtd->cpu_dais[i]; + } else { + dev_err(rtd->card->dev, + "N cpus to M codecs link is not supported yet\n"); + return -EINVAL; + } - for_each_rtd_codec_dai(rtd, i, codec_dai) { if (snd_soc_dai_stream_valid(codec_dai, SNDRV_PCM_STREAM_PLAYBACK) && - snd_soc_dai_stream_valid(cpu_dai, SNDRV_PCM_STREAM_CAPTURE)) + snd_soc_dai_stream_valid(cpu_dai, cpu_playback)) playback = 1; if (snd_soc_dai_stream_valid(codec_dai, SNDRV_PCM_STREAM_CAPTURE) && - snd_soc_dai_stream_valid(cpu_dai, SNDRV_PCM_STREAM_PLAYBACK)) + snd_soc_dai_stream_valid(cpu_dai, cpu_capture)) capture = 1; } - - capture = capture && cpu_capture->channels_min; - playback = playback && cpu_playback->channels_min; } if (rtd->dai_link->playback_only) { @@ -3026,7 +3058,7 @@ int soc_new_pcm(struct snd_soc_pcm_runtime *rtd, int num) out: dev_info(rtd->card->dev, "%s <-> %s mapping ok\n", (rtd->num_codecs > 1) ? "multicodec" : rtd->codec_dai->name, - cpu_dai->name); + (rtd->num_cpus > 1) ? "multicpu" : rtd->cpu_dai->name); return ret; } @@ -3059,33 +3091,17 @@ struct snd_pcm_substream * } EXPORT_SYMBOL_GPL(snd_soc_dpcm_get_substream); -/* get the BE runtime state */ -enum snd_soc_dpcm_state - snd_soc_dpcm_be_get_state(struct snd_soc_pcm_runtime *be, int stream) -{ - return be->dpcm[stream].state; -} -EXPORT_SYMBOL_GPL(snd_soc_dpcm_be_get_state); - -/* set the BE runtime state */ -void snd_soc_dpcm_be_set_state(struct snd_soc_pcm_runtime *be, - int stream, enum snd_soc_dpcm_state state) -{ - be->dpcm[stream].state = state; -} -EXPORT_SYMBOL_GPL(snd_soc_dpcm_be_set_state); - -/* - * We can only hw_free, stop, pause or suspend a BE DAI if any of it's FE - * are not running, paused or suspended for the specified stream direction. - */ -int snd_soc_dpcm_can_be_free_stop(struct snd_soc_pcm_runtime *fe, - struct snd_soc_pcm_runtime *be, int stream) +static int snd_soc_dpcm_check_state(struct snd_soc_pcm_runtime *fe, + struct snd_soc_pcm_runtime *be, + int stream, + const enum snd_soc_dpcm_state *states, + int num_states) { struct snd_soc_dpcm *dpcm; int state; int ret = 1; unsigned long flags; + int i; spin_lock_irqsave(&fe->card->dpcm_lock, flags); for_each_dpcm_fe(be, stream, dpcm) { @@ -3094,18 +3110,34 @@ int snd_soc_dpcm_can_be_free_stop(struct snd_soc_pcm_runtime *fe, continue; state = dpcm->fe->dpcm[stream].state; - if (state == SND_SOC_DPCM_STATE_START || - state == SND_SOC_DPCM_STATE_PAUSED || - state == SND_SOC_DPCM_STATE_SUSPEND) { - ret = 0; - break; + for (i = 0; i < num_states; i++) { + if (state == states[i]) { + ret = 0; + break; + } } } spin_unlock_irqrestore(&fe->card->dpcm_lock, flags); - /* it's safe to free/stop this BE DAI */ + /* it's safe to do this BE DAI */ return ret; } + +/* + * We can only hw_free, stop, pause or suspend a BE DAI if any of it's FE + * are not running, paused or suspended for the specified stream direction. + */ +int snd_soc_dpcm_can_be_free_stop(struct snd_soc_pcm_runtime *fe, + struct snd_soc_pcm_runtime *be, int stream) +{ + const enum snd_soc_dpcm_state state[] = { + SND_SOC_DPCM_STATE_START, + SND_SOC_DPCM_STATE_PAUSED, + SND_SOC_DPCM_STATE_SUSPEND, + }; + + return snd_soc_dpcm_check_state(fe, be, stream, state, ARRAY_SIZE(state)); +} EXPORT_SYMBOL_GPL(snd_soc_dpcm_can_be_free_stop); /* @@ -3115,168 +3147,13 @@ EXPORT_SYMBOL_GPL(snd_soc_dpcm_can_be_free_stop); int snd_soc_dpcm_can_be_params(struct snd_soc_pcm_runtime *fe, struct snd_soc_pcm_runtime *be, int stream) { - struct snd_soc_dpcm *dpcm; - int state; - int ret = 1; - unsigned long flags; + const enum snd_soc_dpcm_state state[] = { + SND_SOC_DPCM_STATE_START, + SND_SOC_DPCM_STATE_PAUSED, + SND_SOC_DPCM_STATE_SUSPEND, + SND_SOC_DPCM_STATE_PREPARE, + }; - spin_lock_irqsave(&fe->card->dpcm_lock, flags); - for_each_dpcm_fe(be, stream, dpcm) { - - if (dpcm->fe == fe) - continue; - - state = dpcm->fe->dpcm[stream].state; - if (state == SND_SOC_DPCM_STATE_START || - state == SND_SOC_DPCM_STATE_PAUSED || - state == SND_SOC_DPCM_STATE_SUSPEND || - state == SND_SOC_DPCM_STATE_PREPARE) { - ret = 0; - break; - } - } - spin_unlock_irqrestore(&fe->card->dpcm_lock, flags); - - /* it's safe to change hw_params */ - return ret; + return snd_soc_dpcm_check_state(fe, be, stream, state, ARRAY_SIZE(state)); } EXPORT_SYMBOL_GPL(snd_soc_dpcm_can_be_params); - -#ifdef CONFIG_DEBUG_FS -static const char *dpcm_state_string(enum snd_soc_dpcm_state state) -{ - switch (state) { - case SND_SOC_DPCM_STATE_NEW: - return "new"; - case SND_SOC_DPCM_STATE_OPEN: - return "open"; - case SND_SOC_DPCM_STATE_HW_PARAMS: - return "hw_params"; - case SND_SOC_DPCM_STATE_PREPARE: - return "prepare"; - case SND_SOC_DPCM_STATE_START: - return "start"; - case SND_SOC_DPCM_STATE_STOP: - return "stop"; - case SND_SOC_DPCM_STATE_SUSPEND: - return "suspend"; - case SND_SOC_DPCM_STATE_PAUSED: - return "paused"; - case SND_SOC_DPCM_STATE_HW_FREE: - return "hw_free"; - case SND_SOC_DPCM_STATE_CLOSE: - return "close"; - } - - return "unknown"; -} - -static ssize_t dpcm_show_state(struct snd_soc_pcm_runtime *fe, - int stream, char *buf, size_t size) -{ - struct snd_pcm_hw_params *params = &fe->dpcm[stream].hw_params; - struct snd_soc_dpcm *dpcm; - ssize_t offset = 0; - unsigned long flags; - - /* FE state */ - offset += scnprintf(buf + offset, size - offset, - "[%s - %s]\n", fe->dai_link->name, - stream ? "Capture" : "Playback"); - - offset += scnprintf(buf + offset, size - offset, "State: %s\n", - dpcm_state_string(fe->dpcm[stream].state)); - - if ((fe->dpcm[stream].state >= SND_SOC_DPCM_STATE_HW_PARAMS) && - (fe->dpcm[stream].state <= SND_SOC_DPCM_STATE_STOP)) - offset += scnprintf(buf + offset, size - offset, - "Hardware Params: " - "Format = %s, Channels = %d, Rate = %d\n", - snd_pcm_format_name(params_format(params)), - params_channels(params), - params_rate(params)); - - /* BEs state */ - offset += scnprintf(buf + offset, size - offset, "Backends:\n"); - - if (list_empty(&fe->dpcm[stream].be_clients)) { - offset += scnprintf(buf + offset, size - offset, - " No active DSP links\n"); - goto out; - } - - spin_lock_irqsave(&fe->card->dpcm_lock, flags); - for_each_dpcm_be(fe, stream, dpcm) { - struct snd_soc_pcm_runtime *be = dpcm->be; - params = &dpcm->hw_params; - - offset += scnprintf(buf + offset, size - offset, - "- %s\n", be->dai_link->name); - - offset += scnprintf(buf + offset, size - offset, - " State: %s\n", - dpcm_state_string(be->dpcm[stream].state)); - - if ((be->dpcm[stream].state >= SND_SOC_DPCM_STATE_HW_PARAMS) && - (be->dpcm[stream].state <= SND_SOC_DPCM_STATE_STOP)) - offset += scnprintf(buf + offset, size - offset, - " Hardware Params: " - "Format = %s, Channels = %d, Rate = %d\n", - snd_pcm_format_name(params_format(params)), - params_channels(params), - params_rate(params)); - } - spin_unlock_irqrestore(&fe->card->dpcm_lock, flags); -out: - return offset; -} - -static ssize_t dpcm_state_read_file(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) -{ - struct snd_soc_pcm_runtime *fe = file->private_data; - ssize_t out_count = PAGE_SIZE, offset = 0, ret = 0; - char *buf; - - buf = kmalloc(out_count, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - if (snd_soc_dai_stream_valid(fe->cpu_dai, SNDRV_PCM_STREAM_PLAYBACK)) - offset += dpcm_show_state(fe, SNDRV_PCM_STREAM_PLAYBACK, - buf + offset, out_count - offset); - - if (snd_soc_dai_stream_valid(fe->cpu_dai, SNDRV_PCM_STREAM_CAPTURE)) - offset += dpcm_show_state(fe, SNDRV_PCM_STREAM_CAPTURE, - buf + offset, out_count - offset); - - ret = simple_read_from_buffer(user_buf, count, ppos, buf, offset); - - kfree(buf); - return ret; -} - -static const struct file_operations dpcm_state_fops = { - .open = simple_open, - .read = dpcm_state_read_file, - .llseek = default_llseek, -}; - -void soc_dpcm_debugfs_add(struct snd_soc_pcm_runtime *rtd) -{ - if (!rtd->dai_link) - return; - - if (!rtd->dai_link->dynamic) - return; - - if (!rtd->card->debugfs_card_root) - return; - - rtd->debugfs_dpcm_root = debugfs_create_dir(rtd->dai_link->name, - rtd->card->debugfs_card_root); - - debugfs_create_file("state", 0444, rtd->debugfs_dpcm_root, - rtd, &dpcm_state_fops); -} -#endif diff --git a/sound/soc/soc-topology.c b/sound/soc/soc-topology.c index 575da6aba807..1f81cd2d29cf 100644 --- a/sound/soc/soc-topology.c +++ b/sound/soc/soc-topology.c @@ -251,7 +251,7 @@ static int soc_tplg_vendor_load_(struct soc_tplg *tplg, { int ret = 0; - if (tplg->comp && tplg->ops && tplg->ops->vendor_load) + if (tplg->ops && tplg->ops->vendor_load) ret = tplg->ops->vendor_load(tplg->comp, tplg->index, hdr); else { dev_err(tplg->dev, "ASoC: no vendor load callback for ID %d\n", @@ -283,7 +283,7 @@ static int soc_tplg_vendor_load(struct soc_tplg *tplg, static int soc_tplg_widget_load(struct soc_tplg *tplg, struct snd_soc_dapm_widget *w, struct snd_soc_tplg_dapm_widget *tplg_w) { - if (tplg->comp && tplg->ops && tplg->ops->widget_load) + if (tplg->ops && tplg->ops->widget_load) return tplg->ops->widget_load(tplg->comp, tplg->index, w, tplg_w); @@ -295,7 +295,7 @@ static int soc_tplg_widget_load(struct soc_tplg *tplg, static int soc_tplg_widget_ready(struct soc_tplg *tplg, struct snd_soc_dapm_widget *w, struct snd_soc_tplg_dapm_widget *tplg_w) { - if (tplg->comp && tplg->ops && tplg->ops->widget_ready) + if (tplg->ops && tplg->ops->widget_ready) return tplg->ops->widget_ready(tplg->comp, tplg->index, w, tplg_w); @@ -307,7 +307,7 @@ static int soc_tplg_dai_load(struct soc_tplg *tplg, struct snd_soc_dai_driver *dai_drv, struct snd_soc_tplg_pcm *pcm, struct snd_soc_dai *dai) { - if (tplg->comp && tplg->ops && tplg->ops->dai_load) + if (tplg->ops && tplg->ops->dai_load) return tplg->ops->dai_load(tplg->comp, tplg->index, dai_drv, pcm, dai); @@ -318,7 +318,7 @@ static int soc_tplg_dai_load(struct soc_tplg *tplg, static int soc_tplg_dai_link_load(struct soc_tplg *tplg, struct snd_soc_dai_link *link, struct snd_soc_tplg_link_config *cfg) { - if (tplg->comp && tplg->ops && tplg->ops->link_load) + if (tplg->ops && tplg->ops->link_load) return tplg->ops->link_load(tplg->comp, tplg->index, link, cfg); return 0; @@ -327,7 +327,7 @@ static int soc_tplg_dai_link_load(struct soc_tplg *tplg, /* tell the component driver that all firmware has been loaded in this request */ static void soc_tplg_complete(struct soc_tplg *tplg) { - if (tplg->comp && tplg->ops && tplg->ops->complete) + if (tplg->ops && tplg->ops->complete) tplg->ops->complete(tplg->comp); } @@ -684,7 +684,7 @@ EXPORT_SYMBOL_GPL(snd_soc_tplg_widget_bind_event); static int soc_tplg_init_kcontrol(struct soc_tplg *tplg, struct snd_kcontrol_new *k, struct snd_soc_tplg_ctl_hdr *hdr) { - if (tplg->comp && tplg->ops && tplg->ops->control_load) + if (tplg->ops && tplg->ops->control_load) return tplg->ops->control_load(tplg->comp, tplg->index, k, hdr); @@ -1174,7 +1174,7 @@ static int soc_tplg_kcontrol_elems_load(struct soc_tplg *tplg, static int soc_tplg_add_route(struct soc_tplg *tplg, struct snd_soc_dapm_route *route) { - if (tplg->comp && tplg->ops && tplg->ops->dapm_route_load) + if (tplg->ops && tplg->ops->dapm_route_load) return tplg->ops->dapm_route_load(tplg->comp, tplg->index, route); @@ -2564,7 +2564,7 @@ static int soc_tplg_manifest_load(struct soc_tplg *tplg, } /* pass control to component driver for optional further init */ - if (tplg->comp && tplg->ops && tplg->ops->manifest) + if (tplg->ops && tplg->ops->manifest) ret = tplg->ops->manifest(tplg->comp, tplg->index, _manifest); if (!abi_match) /* free the duplicated one */ @@ -2736,6 +2736,10 @@ int snd_soc_tplg_component_load(struct snd_soc_component *comp, struct soc_tplg tplg; int ret; + /* component needs to exist to keep and reference data while parsing */ + if (!comp) + return -EINVAL; + /* setup parsing context */ memset(&tplg, 0, sizeof(tplg)); tplg.fw = fw; @@ -2774,7 +2778,7 @@ void snd_soc_tplg_widget_remove_all(struct snd_soc_dapm_context *dapm, { struct snd_soc_dapm_widget *w, *next_w; - list_for_each_entry_safe(w, next_w, &dapm->card->widgets, list) { + for_each_card_widgets_safe(dapm->card, w, next_w) { /* make sure we are a widget with correct context */ if (w->dobj.type != SND_SOC_DOBJ_WIDGET || w->dapm != dapm) diff --git a/sound/soc/sof/Kconfig b/sound/soc/sof/Kconfig index 827b0ec92522..4dda4b62509f 100644 --- a/sound/soc/sof/Kconfig +++ b/sound/soc/sof/Kconfig @@ -41,6 +41,15 @@ config SND_SOC_SOF_OF required to enable i.MX8 devices. Say Y if you need this option. If unsure select "N". +config SND_SOC_SOF_DEBUG_PROBES + bool "SOF enable data probing" + select SND_SOC_COMPRESS + help + This option enables the data probing feature that can be used to + gather data directly from specific points of the audio pipeline. + Say Y if you want to enable probes. + If unsure, select "N". + config SND_SOC_SOF_DEVELOPER_SUPPORT bool "SOF developer options support" depends on EXPERT diff --git a/sound/soc/sof/Makefile b/sound/soc/sof/Makefile index 0a8bc72c28a5..8eca2f85c90e 100644 --- a/sound/soc/sof/Makefile +++ b/sound/soc/sof/Makefile @@ -2,6 +2,7 @@ snd-sof-objs := core.o ops.o loader.o ipc.o pcm.o pm.o debug.o topology.o\ control.o trace.o utils.o sof-audio.o +snd-sof-$(CONFIG_SND_SOC_SOF_DEBUG_PROBES) += probe.o compress.o snd-sof-pci-objs := sof-pci-dev.o snd-sof-acpi-objs := sof-acpi-dev.o diff --git a/sound/soc/sof/compress.c b/sound/soc/sof/compress.c new file mode 100644 index 000000000000..7354dc6a49cf --- /dev/null +++ b/sound/soc/sof/compress.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2019-2020 Intel Corporation. All rights reserved. +// +// Author: Cezary Rojewski +// + +#include +#include "compress.h" +#include "ops.h" +#include "probe.h" + +struct snd_compr_ops sof_probe_compressed_ops = { + .copy = sof_probe_compr_copy, +}; +EXPORT_SYMBOL(sof_probe_compressed_ops); + +int sof_probe_compr_open(struct snd_compr_stream *cstream, + struct snd_soc_dai *dai) +{ + struct snd_sof_dev *sdev = + snd_soc_component_get_drvdata(dai->component); + int ret; + + ret = snd_sof_probe_compr_assign(sdev, cstream, dai); + if (ret < 0) { + dev_err(dai->dev, "Failed to assign probe stream: %d\n", ret); + return ret; + } + + sdev->extractor_stream_tag = ret; + return 0; +} +EXPORT_SYMBOL(sof_probe_compr_open); + +int sof_probe_compr_free(struct snd_compr_stream *cstream, + struct snd_soc_dai *dai) +{ + struct snd_sof_dev *sdev = + snd_soc_component_get_drvdata(dai->component); + struct sof_probe_point_desc *desc; + size_t num_desc; + int i, ret; + + /* disconnect all probe points */ + ret = sof_ipc_probe_points_info(sdev, &desc, &num_desc); + if (ret < 0) { + dev_err(dai->dev, "Failed to get probe points: %d\n", ret); + goto exit; + } + + for (i = 0; i < num_desc; i++) + sof_ipc_probe_points_remove(sdev, &desc[i].buffer_id, 1); + kfree(desc); + +exit: + ret = sof_ipc_probe_deinit(sdev); + if (ret < 0) + dev_err(dai->dev, "Failed to deinit probe: %d\n", ret); + + sdev->extractor_stream_tag = SOF_PROBE_INVALID_NODE_ID; + snd_compr_free_pages(cstream); + + return snd_sof_probe_compr_free(sdev, cstream, dai); +} +EXPORT_SYMBOL(sof_probe_compr_free); + +int sof_probe_compr_set_params(struct snd_compr_stream *cstream, + struct snd_compr_params *params, struct snd_soc_dai *dai) +{ + struct snd_compr_runtime *rtd = cstream->runtime; + struct snd_sof_dev *sdev = + snd_soc_component_get_drvdata(dai->component); + int ret; + + cstream->dma_buffer.dev.type = SNDRV_DMA_TYPE_DEV_SG; + cstream->dma_buffer.dev.dev = sdev->dev; + ret = snd_compr_malloc_pages(cstream, rtd->buffer_size); + if (ret < 0) + return ret; + + ret = snd_sof_probe_compr_set_params(sdev, cstream, params, dai); + if (ret < 0) + return ret; + + ret = sof_ipc_probe_init(sdev, sdev->extractor_stream_tag, + rtd->dma_bytes); + if (ret < 0) { + dev_err(dai->dev, "Failed to init probe: %d\n", ret); + return ret; + } + + return 0; +} +EXPORT_SYMBOL(sof_probe_compr_set_params); + +int sof_probe_compr_trigger(struct snd_compr_stream *cstream, int cmd, + struct snd_soc_dai *dai) +{ + struct snd_sof_dev *sdev = + snd_soc_component_get_drvdata(dai->component); + + return snd_sof_probe_compr_trigger(sdev, cstream, cmd, dai); +} +EXPORT_SYMBOL(sof_probe_compr_trigger); + +int sof_probe_compr_pointer(struct snd_compr_stream *cstream, + struct snd_compr_tstamp *tstamp, struct snd_soc_dai *dai) +{ + struct snd_sof_dev *sdev = + snd_soc_component_get_drvdata(dai->component); + + return snd_sof_probe_compr_pointer(sdev, cstream, tstamp, dai); +} +EXPORT_SYMBOL(sof_probe_compr_pointer); + +int sof_probe_compr_copy(struct snd_compr_stream *cstream, + char __user *buf, size_t count) +{ + struct snd_compr_runtime *rtd = cstream->runtime; + unsigned int offset, n; + void *ptr; + int ret; + + if (count > rtd->buffer_size) + count = rtd->buffer_size; + + div_u64_rem(rtd->total_bytes_transferred, rtd->buffer_size, &offset); + ptr = rtd->dma_area + offset; + n = rtd->buffer_size - offset; + + if (count < n) { + ret = copy_to_user(buf, ptr, count); + } else { + ret = copy_to_user(buf, ptr, n); + ret += copy_to_user(buf + n, rtd->dma_area, count - n); + } + + if (ret) + return count - ret; + return count; +} +EXPORT_SYMBOL(sof_probe_compr_copy); diff --git a/sound/soc/sof/compress.h b/sound/soc/sof/compress.h new file mode 100644 index 000000000000..800f163603e1 --- /dev/null +++ b/sound/soc/sof/compress.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * Copyright(c) 2019-2020 Intel Corporation. All rights reserved. + * + * Author: Cezary Rojewski + */ + +#ifndef __SOF_COMPRESS_H +#define __SOF_COMPRESS_H + +#include + +extern struct snd_compr_ops sof_probe_compressed_ops; + +int sof_probe_compr_open(struct snd_compr_stream *cstream, + struct snd_soc_dai *dai); +int sof_probe_compr_free(struct snd_compr_stream *cstream, + struct snd_soc_dai *dai); +int sof_probe_compr_set_params(struct snd_compr_stream *cstream, + struct snd_compr_params *params, struct snd_soc_dai *dai); +int sof_probe_compr_trigger(struct snd_compr_stream *cstream, int cmd, + struct snd_soc_dai *dai); +int sof_probe_compr_pointer(struct snd_compr_stream *cstream, + struct snd_compr_tstamp *tstamp, struct snd_soc_dai *dai); +int sof_probe_compr_copy(struct snd_compr_stream *cstream, + char __user *buf, size_t count); + +#endif diff --git a/sound/soc/sof/core.c b/sound/soc/sof/core.c index 34cefbaf2d2a..91acfae7935c 100644 --- a/sound/soc/sof/core.c +++ b/sound/soc/sof/core.c @@ -14,6 +14,9 @@ #include #include "sof-priv.h" #include "ops.h" +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_PROBES) +#include "probe.h" +#endif /* see SOF_DBG_ flags */ int sof_core_debug; @@ -286,12 +289,15 @@ int snd_sof_device_probe(struct device *dev, struct snd_sof_pdata *plat_data) /* initialize sof device */ sdev->dev = dev; - /* initialize default D0 sub-state */ - sdev->d0_substate = SOF_DSP_D0I0; + /* initialize default DSP power state */ + sdev->dsp_power_state.state = SOF_DSP_PM_D0; sdev->pdata = plat_data; sdev->first_boot = true; sdev->fw_state = SOF_FW_BOOT_NOT_STARTED; +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_PROBES) + sdev->extractor_stream_tag = SOF_PROBE_INVALID_NODE_ID; +#endif dev_set_drvdata(dev, sdev); /* check all mandatory ops */ diff --git a/sound/soc/sof/debug.c b/sound/soc/sof/debug.c index d2b3b99d3a20..b5c0d6cf72cc 100644 --- a/sound/soc/sof/debug.c +++ b/sound/soc/sof/debug.c @@ -17,6 +17,221 @@ #include "sof-priv.h" #include "ops.h" +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_PROBES) +#include "probe.h" + +/** + * strsplit_u32 - Split string into sequence of u32 tokens + * @buf: String to split into tokens. + * @delim: String containing delimiter characters. + * @tkns: Returned u32 sequence pointer. + * @num_tkns: Returned number of tokens obtained. + */ +static int +strsplit_u32(char **buf, const char *delim, u32 **tkns, size_t *num_tkns) +{ + char *s; + u32 *data, *tmp; + size_t count = 0; + size_t cap = 32; + int ret = 0; + + *tkns = NULL; + *num_tkns = 0; + data = kcalloc(cap, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + while ((s = strsep(buf, delim)) != NULL) { + ret = kstrtouint(s, 0, data + count); + if (ret) + goto exit; + if (++count >= cap) { + cap *= 2; + tmp = krealloc(data, cap * sizeof(*data), GFP_KERNEL); + if (!tmp) { + ret = -ENOMEM; + goto exit; + } + data = tmp; + } + } + + if (!count) + goto exit; + *tkns = kmemdup(data, count * sizeof(*data), GFP_KERNEL); + if (*tkns == NULL) { + ret = -ENOMEM; + goto exit; + } + *num_tkns = count; + +exit: + kfree(data); + return ret; +} + +static int tokenize_input(const char __user *from, size_t count, + loff_t *ppos, u32 **tkns, size_t *num_tkns) +{ + char *buf; + int ret; + + buf = kmalloc(count + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = simple_write_to_buffer(buf, count, ppos, from, count); + if (ret != count) { + ret = ret >= 0 ? -EIO : ret; + goto exit; + } + + buf[count] = '\0'; + ret = strsplit_u32((char **)&buf, ",", tkns, num_tkns); +exit: + kfree(buf); + return ret; +} + +static ssize_t probe_points_read(struct file *file, + char __user *to, size_t count, loff_t *ppos) +{ + struct snd_sof_dfsentry *dfse = file->private_data; + struct snd_sof_dev *sdev = dfse->sdev; + struct sof_probe_point_desc *desc; + size_t num_desc, len = 0; + char *buf; + int i, ret; + + if (sdev->extractor_stream_tag == SOF_PROBE_INVALID_NODE_ID) { + dev_warn(sdev->dev, "no extractor stream running\n"); + return -ENOENT; + } + + buf = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = sof_ipc_probe_points_info(sdev, &desc, &num_desc); + if (ret < 0) + goto exit; + + for (i = 0; i < num_desc; i++) { + ret = snprintf(buf + len, PAGE_SIZE - len, + "Id: %#010x Purpose: %d Node id: %#x\n", + desc[i].buffer_id, desc[i].purpose, desc[i].stream_tag); + if (ret < 0) + goto free_desc; + len += ret; + } + + ret = simple_read_from_buffer(to, count, ppos, buf, len); +free_desc: + kfree(desc); +exit: + kfree(buf); + return ret; +} + +static ssize_t probe_points_write(struct file *file, + const char __user *from, size_t count, loff_t *ppos) +{ + struct snd_sof_dfsentry *dfse = file->private_data; + struct snd_sof_dev *sdev = dfse->sdev; + struct sof_probe_point_desc *desc; + size_t num_tkns, bytes; + u32 *tkns; + int ret; + + if (sdev->extractor_stream_tag == SOF_PROBE_INVALID_NODE_ID) { + dev_warn(sdev->dev, "no extractor stream running\n"); + return -ENOENT; + } + + ret = tokenize_input(from, count, ppos, &tkns, &num_tkns); + if (ret < 0) + return ret; + bytes = sizeof(*tkns) * num_tkns; + if (!num_tkns || (bytes % sizeof(*desc))) { + ret = -EINVAL; + goto exit; + } + + desc = (struct sof_probe_point_desc *)tkns; + ret = sof_ipc_probe_points_add(sdev, + desc, bytes / sizeof(*desc)); + if (!ret) + ret = count; +exit: + kfree(tkns); + return ret; +} + +static const struct file_operations probe_points_fops = { + .open = simple_open, + .read = probe_points_read, + .write = probe_points_write, + .llseek = default_llseek, +}; + +static ssize_t probe_points_remove_write(struct file *file, + const char __user *from, size_t count, loff_t *ppos) +{ + struct snd_sof_dfsentry *dfse = file->private_data; + struct snd_sof_dev *sdev = dfse->sdev; + size_t num_tkns; + u32 *tkns; + int ret; + + if (sdev->extractor_stream_tag == SOF_PROBE_INVALID_NODE_ID) { + dev_warn(sdev->dev, "no extractor stream running\n"); + return -ENOENT; + } + + ret = tokenize_input(from, count, ppos, &tkns, &num_tkns); + if (ret < 0) + return ret; + if (!num_tkns) { + ret = -EINVAL; + goto exit; + } + + ret = sof_ipc_probe_points_remove(sdev, tkns, num_tkns); + if (!ret) + ret = count; +exit: + kfree(tkns); + return ret; +} + +static const struct file_operations probe_points_remove_fops = { + .open = simple_open, + .write = probe_points_remove_write, + .llseek = default_llseek, +}; + +static int snd_sof_debugfs_probe_item(struct snd_sof_dev *sdev, + const char *name, mode_t mode, + const struct file_operations *fops) +{ + struct snd_sof_dfsentry *dfse; + + dfse = devm_kzalloc(sdev->dev, sizeof(*dfse), GFP_KERNEL); + if (!dfse) + return -ENOMEM; + + dfse->type = SOF_DFSENTRY_TYPE_BUF; + dfse->sdev = sdev; + + debugfs_create_file(name, mode, sdev->debugfs_root, dfse, fops); + /* add to dfsentry list */ + list_add(&dfse->list, &sdev->dfsentry_list); + + return 0; +} +#endif + #if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_IPC_FLOOD_TEST) #define MAX_IPC_FLOOD_DURATION_MS 1000 #define MAX_IPC_FLOOD_COUNT 10000 @@ -436,6 +651,17 @@ int snd_sof_dbg_init(struct snd_sof_dev *sdev) return err; } +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_PROBES) + err = snd_sof_debugfs_probe_item(sdev, "probe_points", + 0644, &probe_points_fops); + if (err < 0) + return err; + err = snd_sof_debugfs_probe_item(sdev, "probe_points_remove", + 0200, &probe_points_remove_fops); + if (err < 0) + return err; +#endif + #if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_IPC_FLOOD_TEST) /* create read-write ipc_flood_count debugfs entry */ err = snd_sof_debugfs_buf_item(sdev, NULL, 0, diff --git a/sound/soc/sof/imx/imx8.c b/sound/soc/sof/imx/imx8.c index b2556f5e2871..b692752b2178 100644 --- a/sound/soc/sof/imx/imx8.c +++ b/sound/soc/sof/imx/imx8.c @@ -138,7 +138,7 @@ static int imx8_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) /* * DSP control. */ -static int imx8_run(struct snd_sof_dev *sdev) +static int imx8x_run(struct snd_sof_dev *sdev) { struct imx8_priv *dsp_priv = (struct imx8_priv *)sdev->private; int ret; @@ -178,6 +178,24 @@ static int imx8_run(struct snd_sof_dev *sdev) return 0; } +static int imx8_run(struct snd_sof_dev *sdev) +{ + struct imx8_priv *dsp_priv = (struct imx8_priv *)sdev->private; + int ret; + + ret = imx_sc_misc_set_control(dsp_priv->sc_ipc, IMX_SC_R_DSP, + IMX_SC_C_OFS_SEL, 0); + if (ret < 0) { + dev_err(sdev->dev, "Error system address offset source select\n"); + return ret; + } + + imx_sc_pm_cpu_start(dsp_priv->sc_ipc, IMX_SC_R_DSP, true, + RESET_VECTOR_VADDR); + + return 0; +} + static int imx8_probe(struct snd_sof_dev *sdev) { struct platform_device *pdev = @@ -360,7 +378,7 @@ static struct snd_soc_dai_driver imx8_dai[] = { }, }; -/* i.MX8 ops */ +/* i.MX8 ops */ struct snd_sof_dsp_ops sof_imx8_ops = { /* probe and remove */ .probe = imx8_probe, @@ -390,6 +408,39 @@ struct snd_sof_dsp_ops sof_imx8_ops = { /* DAI drivers */ .drv = imx8_dai, .num_drv = 1, /* we have only 1 ESAI interface on i.MX8 */ +}; +EXPORT_SYMBOL(sof_imx8_ops); + +/* i.MX8X ops */ +struct snd_sof_dsp_ops sof_imx8x_ops = { + /* probe and remove */ + .probe = imx8_probe, + .remove = imx8_remove, + /* DSP core boot */ + .run = imx8x_run, + + /* Block IO */ + .block_read = sof_block_read, + .block_write = sof_block_write, + + /* ipc */ + .send_msg = imx8_send_msg, + .fw_ready = sof_fw_ready, + .get_mailbox_offset = imx8_get_mailbox_offset, + .get_window_offset = imx8_get_window_offset, + + .ipc_msg_data = imx8_ipc_msg_data, + .ipc_pcm_params = imx8_ipc_pcm_params, + + /* module loading */ + .load_module = snd_sof_parse_module_memcpy, + .get_bar_index = imx8_get_bar_index, + /* firmware loading */ + .load_firmware = snd_sof_load_firmware_memcpy, + + /* DAI drivers */ + .drv = imx8_dai, + .num_drv = 1, /* we have only 1 ESAI interface on i.MX8 */ /* ALSA HW info flags */ .hw_info = SNDRV_PCM_INFO_MMAP | @@ -398,6 +449,6 @@ struct snd_sof_dsp_ops sof_imx8_ops = { SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_NO_PERIOD_WAKEUP }; -EXPORT_SYMBOL(sof_imx8_ops); +EXPORT_SYMBOL(sof_imx8x_ops); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig index 56a837d2cb95..c9a2bee4b55c 100644 --- a/sound/soc/sof/intel/Kconfig +++ b/sound/soc/sof/intel/Kconfig @@ -305,6 +305,15 @@ config SND_SOC_SOF_HDA_AUDIO_CODEC Say Y if you want to enable HDAudio codecs with SOF. If unsure select "N". +config SND_SOC_SOF_HDA_PROBES + bool "SOF enable probes over HDA" + depends on SND_SOC_SOF_DEBUG_PROBES + help + This option enables the data probing for Intel(R). + Intel(R) Skylake and newer platforms. + Say Y if you want to enable probes. + If unsure, select "N". + config SND_SOC_SOF_HDA_ALWAYS_ENABLE_DMI_L1 bool "SOF enable DMI Link L1" help @@ -315,17 +324,6 @@ config SND_SOC_SOF_HDA_ALWAYS_ENABLE_DMI_L1 Say Y if you want to enable DMI Link L1 If unsure, select "N". -config SND_SOC_SOF_HDA_COMMON_HDMI_CODEC - bool "SOF common HDA HDMI codec driver" - depends on SND_SOC_SOF_HDA_LINK - depends on SND_HDA_CODEC_HDMI - default SND_HDA_CODEC_HDMI - help - This adds support for HDMI audio by using the common HDA - HDMI/DisplayPort codec driver. - Say Y if you want to use the common codec driver with SOF. - If unsure select "Y". - endif ## SND_SOC_SOF_HDA_COMMON config SND_SOC_SOF_HDA_LINK_BASELINE diff --git a/sound/soc/sof/intel/Makefile b/sound/soc/sof/intel/Makefile index b8f58e006e29..cee02a2e00f4 100644 --- a/sound/soc/sof/intel/Makefile +++ b/sound/soc/sof/intel/Makefile @@ -9,6 +9,7 @@ snd-sof-intel-hda-common-objs := hda.o hda-loader.o hda-stream.o hda-trace.o \ hda-dsp.o hda-ipc.o hda-ctrl.o hda-pcm.o \ hda-dai.o hda-bus.o \ apl.o cnl.o +snd-sof-intel-hda-common-$(CONFIG_SND_SOC_SOF_HDA_PROBES) += hda-compress.o snd-sof-intel-hda-objs := hda-codec.o diff --git a/sound/soc/sof/intel/apl.c b/sound/soc/sof/intel/apl.c index 2483b15699e7..02218d22e51f 100644 --- a/sound/soc/sof/intel/apl.c +++ b/sound/soc/sof/intel/apl.c @@ -73,6 +73,15 @@ const struct snd_sof_dsp_ops sof_apl_ops = { .pcm_trigger = hda_dsp_pcm_trigger, .pcm_pointer = hda_dsp_pcm_pointer, +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_PROBES) + /* probe callbacks */ + .probe_assign = hda_probe_compr_assign, + .probe_free = hda_probe_compr_free, + .probe_set_params = hda_probe_compr_set_params, + .probe_trigger = hda_probe_compr_trigger, + .probe_pointer = hda_probe_compr_pointer, +#endif + /* firmware loading */ .load_firmware = snd_sof_load_firmware_raw, diff --git a/sound/soc/sof/intel/cnl.c b/sound/soc/sof/intel/cnl.c index 9e2d8afe0535..e427d00eca71 100644 --- a/sound/soc/sof/intel/cnl.c +++ b/sound/soc/sof/intel/cnl.c @@ -65,11 +65,6 @@ static irqreturn_t cnl_ipc_irq_thread(int irq, void *context) hda_dsp_ipc_get_reply(sdev); snd_sof_ipc_reply(sdev, msg); - if (sdev->code_loading) { - sdev->code_loading = 0; - wake_up(&sdev->waitq); - } - cnl_ipc_dsp_done(sdev); spin_unlock_irq(&sdev->ipc_lock); @@ -171,23 +166,48 @@ static bool cnl_compact_ipc_compress(struct snd_sof_ipc_msg *msg, static int cnl_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) { + struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; + struct sof_ipc_cmd_hdr *hdr; u32 dr = 0; u32 dd = 0; + /* + * Currently the only compact IPC supported is the PM_GATE + * IPC which is used for transitioning the DSP between the + * D0I0 and D0I3 states. And these are sent only during the + * set_power_state() op. Therefore, there will never be a case + * that a compact IPC results in the DSP exiting D0I3 without + * the host and FW being in sync. + */ if (cnl_compact_ipc_compress(msg, &dr, &dd)) { /* send the message via IPC registers */ snd_sof_dsp_write(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDD, dd); snd_sof_dsp_write(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDR, CNL_DSP_REG_HIPCIDR_BUSY | dr); - } else { - /* send the message via mailbox */ - sof_mailbox_write(sdev, sdev->host_box.offset, msg->msg_data, - msg->msg_size); - snd_sof_dsp_write(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDR, - CNL_DSP_REG_HIPCIDR_BUSY); + return 0; } + /* send the message via mailbox */ + sof_mailbox_write(sdev, sdev->host_box.offset, msg->msg_data, + msg->msg_size); + snd_sof_dsp_write(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDR, + CNL_DSP_REG_HIPCIDR_BUSY); + + hdr = msg->msg_data; + + /* + * Use mod_delayed_work() to schedule the delayed work + * to avoid scheduling multiple workqueue items when + * IPCs are sent at a high-rate. mod_delayed_work() + * modifies the timer if the work is pending. + * Also, a new delayed work should not be queued after the + * the CTX_SAVE IPC, which is sent before the DSP enters D3. + */ + if (hdr->cmd != (SOF_IPC_GLB_PM_MSG | SOF_IPC_PM_CTX_SAVE)) + mod_delayed_work(system_wq, &hdev->d0i3_work, + msecs_to_jiffies(SOF_HDA_D0I3_WORK_DELAY_MS)); + return 0; } @@ -259,6 +279,15 @@ const struct snd_sof_dsp_ops sof_cnl_ops = { .pcm_trigger = hda_dsp_pcm_trigger, .pcm_pointer = hda_dsp_pcm_pointer, +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_PROBES) + /* probe callbacks */ + .probe_assign = hda_probe_compr_assign, + .probe_free = hda_probe_compr_free, + .probe_set_params = hda_probe_compr_set_params, + .probe_trigger = hda_probe_compr_trigger, + .probe_pointer = hda_probe_compr_pointer, +#endif + /* firmware loading */ .load_firmware = snd_sof_load_firmware_raw, diff --git a/sound/soc/sof/intel/hda-codec.c b/sound/soc/sof/intel/hda-codec.c index ff45075ef720..3041fbbb010a 100644 --- a/sound/soc/sof/intel/hda-codec.c +++ b/sound/soc/sof/intel/hda-codec.c @@ -113,8 +113,14 @@ static int hda_codec_probe(struct snd_sof_dev *sdev, int address, if (ret < 0) return ret; - if ((resp & 0xFFFF0000) == IDISP_VID_INTEL) + if ((resp & 0xFFFF0000) == IDISP_VID_INTEL) { + if (!hdev->bus->audio_component) { + dev_dbg(sdev->dev, + "iDisp hw present but no driver\n"); + return -ENOENT; + } hda_priv->need_display_power = true; + } /* * if common HDMI codec driver is not used, codec load @@ -203,6 +209,9 @@ int hda_codec_i915_exit(struct snd_sof_dev *sdev) struct hdac_bus *bus = sof_to_bus(sdev); int ret; + if (!bus->audio_component) + return 0; + /* power down unconditionally */ snd_hdac_display_power(bus, HDA_CODEC_IDX_CONTROLLER, false); diff --git a/sound/soc/sof/intel/hda-compress.c b/sound/soc/sof/intel/hda-compress.c new file mode 100644 index 000000000000..38a1ebec8478 --- /dev/null +++ b/sound/soc/sof/intel/hda-compress.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2019-2020 Intel Corporation. All rights reserved. +// +// Author: Cezary Rojewski +// + +#include +#include +#include "../sof-priv.h" +#include "hda.h" + +static inline struct hdac_ext_stream * +hda_compr_get_stream(struct snd_compr_stream *cstream) +{ + return cstream->runtime->private_data; +} + +int hda_probe_compr_assign(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_soc_dai *dai) +{ + struct hdac_ext_stream *stream; + + stream = hda_dsp_stream_get(sdev, cstream->direction); + if (!stream) + return -EBUSY; + + hdac_stream(stream)->curr_pos = 0; + hdac_stream(stream)->cstream = cstream; + cstream->runtime->private_data = stream; + + return hdac_stream(stream)->stream_tag; +} + +int hda_probe_compr_free(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_soc_dai *dai) +{ + struct hdac_ext_stream *stream = hda_compr_get_stream(cstream); + int ret; + + ret = hda_dsp_stream_put(sdev, cstream->direction, + hdac_stream(stream)->stream_tag); + if (ret < 0) { + dev_dbg(sdev->dev, "stream put failed: %d\n", ret); + return ret; + } + + hdac_stream(stream)->cstream = NULL; + cstream->runtime->private_data = NULL; + + return 0; +} + +int hda_probe_compr_set_params(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_compr_params *params, + struct snd_soc_dai *dai) +{ + struct hdac_ext_stream *stream = hda_compr_get_stream(cstream); + struct hdac_stream *hstream = hdac_stream(stream); + struct snd_dma_buffer *dmab; + u32 bits, rate; + int bps, ret; + + dmab = cstream->runtime->dma_buffer_p; + /* compr params do not store bit depth, default to S32_LE */ + bps = snd_pcm_format_physical_width(SNDRV_PCM_FORMAT_S32_LE); + if (bps < 0) + return bps; + bits = hda_dsp_get_bits(sdev, bps); + rate = hda_dsp_get_mult_div(sdev, params->codec.sample_rate); + + hstream->format_val = rate | bits | (params->codec.ch_out - 1); + hstream->bufsize = cstream->runtime->buffer_size; + hstream->period_bytes = cstream->runtime->fragment_size; + hstream->no_period_wakeup = 0; + + ret = hda_dsp_stream_hw_params(sdev, stream, dmab, NULL); + if (ret < 0) { + dev_err(sdev->dev, "error: hdac prepare failed: %x\n", ret); + return ret; + } + + return 0; +} + +int hda_probe_compr_trigger(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, int cmd, + struct snd_soc_dai *dai) +{ + struct hdac_ext_stream *stream = hda_compr_get_stream(cstream); + + return hda_dsp_stream_trigger(sdev, stream, cmd); +} + +int hda_probe_compr_pointer(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_compr_tstamp *tstamp, + struct snd_soc_dai *dai) +{ + struct hdac_ext_stream *stream = hda_compr_get_stream(cstream); + struct snd_soc_pcm_stream *pstream; + + pstream = &dai->driver->capture; + tstamp->copied_total = hdac_stream(stream)->curr_pos; + tstamp->sampling_rate = snd_pcm_rate_bit_to_rate(pstream->rates); + + return 0; +} diff --git a/sound/soc/sof/intel/hda-ctrl.c b/sound/soc/sof/intel/hda-ctrl.c index 871b71a15a63..6288b2f99540 100644 --- a/sound/soc/sof/intel/hda-ctrl.c +++ b/sound/soc/sof/intel/hda-ctrl.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "../ops.h" #include "hda.h" @@ -64,15 +65,32 @@ int hda_dsp_ctrl_get_caps(struct snd_sof_dev *sdev) struct hdac_bus *bus = sof_to_bus(sdev); u32 cap, offset, feature; int count = 0; + int ret; + + /* + * On some devices, one reset cycle is necessary before reading + * capabilities + */ + ret = hda_dsp_ctrl_link_reset(sdev, true); + if (ret < 0) + return ret; + ret = hda_dsp_ctrl_link_reset(sdev, false); + if (ret < 0) + return ret; offset = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, SOF_HDA_LLCH); do { - cap = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, offset); - dev_dbg(sdev->dev, "checking for capabilities at offset 0x%x\n", offset & SOF_HDA_CAP_NEXT_MASK); + cap = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, offset); + + if (cap == -1) { + dev_dbg(bus->dev, "Invalid capability reg read\n"); + break; + } + feature = (cap & SOF_HDA_CAP_ID_MASK) >> SOF_HDA_CAP_ID_OFF; switch (feature) { @@ -105,8 +123,8 @@ int hda_dsp_ctrl_get_caps(struct snd_sof_dev *sdev) bus->mlcap = bus->remap_addr + offset; break; default: - dev_vdbg(sdev->dev, "found capability %d at 0x%x\n", - feature, offset); + dev_dbg(sdev->dev, "found capability %d at 0x%x\n", + feature, offset); break; } @@ -176,6 +194,9 @@ int hda_dsp_ctrl_init_chip(struct snd_sof_dev *sdev, bool full_reset) if (bus->chip_init) return 0; +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + snd_hdac_set_codec_wakeup(bus, true); +#endif hda_dsp_ctrl_misc_clock_gating(sdev, false); if (full_reset) { @@ -183,7 +204,7 @@ int hda_dsp_ctrl_init_chip(struct snd_sof_dev *sdev, bool full_reset) ret = hda_dsp_ctrl_link_reset(sdev, true); if (ret < 0) { dev_err(sdev->dev, "error: failed to reset HDA controller\n"); - return ret; + goto err; } usleep_range(500, 1000); @@ -192,7 +213,7 @@ int hda_dsp_ctrl_init_chip(struct snd_sof_dev *sdev, bool full_reset) ret = hda_dsp_ctrl_link_reset(sdev, false); if (ret < 0) { dev_err(sdev->dev, "error: failed to exit HDA controller reset\n"); - return ret; + goto err; } usleep_range(1000, 1200); @@ -202,7 +223,8 @@ int hda_dsp_ctrl_init_chip(struct snd_sof_dev *sdev, bool full_reset) /* check to see if controller is ready */ if (!snd_hdac_chip_readb(bus, GCTL)) { dev_dbg(bus->dev, "controller not ready!\n"); - return -EBUSY; + ret = -EBUSY; + goto err; } /* Accept unsolicited responses */ @@ -268,7 +290,11 @@ int hda_dsp_ctrl_init_chip(struct snd_sof_dev *sdev, bool full_reset) bus->chip_init = true; +err: hda_dsp_ctrl_misc_clock_gating(sdev, true); +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + snd_hdac_set_codec_wakeup(bus, false); +#endif return ret; } diff --git a/sound/soc/sof/intel/hda-dai.c b/sound/soc/sof/intel/hda-dai.c index 9c6e3f990ee3..833dc303b394 100644 --- a/sound/soc/sof/intel/hda-dai.c +++ b/sound/soc/sof/intel/hda-dai.c @@ -204,7 +204,7 @@ static int hda_link_hw_params(struct snd_pcm_substream *substream, struct hdac_bus *bus = hstream->bus; struct hdac_ext_stream *link_dev; struct snd_soc_pcm_runtime *rtd = snd_pcm_substream_chip(substream); - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct sof_intel_hda_stream *hda_stream; struct hda_pipe_params p_params = {0}; struct hdac_ext_link *link; @@ -293,7 +293,7 @@ static int hda_link_pcm_trigger(struct snd_pcm_substream *substream, bus = hstream->bus; rtd = snd_pcm_substream_chip(substream); - link = snd_hdac_ext_bus_get_link(bus, rtd->codec_dai->component->name); + link = snd_hdac_ext_bus_get_link(bus, asoc_rtd_to_codec(rtd, 0)->component->name); if (!link) return -EINVAL; @@ -374,7 +374,7 @@ static int hda_link_hw_free(struct snd_pcm_substream *substream, if (ret < 0) return ret; - link = snd_hdac_ext_bus_get_link(bus, rtd->codec_dai->component->name); + link = snd_hdac_ext_bus_get_link(bus, asoc_rtd_to_codec(rtd, 0)->component->name); if (!link) return -EINVAL; @@ -399,6 +399,19 @@ static const struct snd_soc_dai_ops hda_link_dai_ops = { .trigger = hda_link_pcm_trigger, .prepare = hda_link_pcm_prepare, }; + +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_PROBES) +#include "../compress.h" + +static struct snd_soc_cdai_ops sof_probe_compr_ops = { + .startup = sof_probe_compr_open, + .shutdown = sof_probe_compr_free, + .set_params = sof_probe_compr_set_params, + .trigger = sof_probe_compr_trigger, + .pointer = sof_probe_compr_pointer, +}; + +#endif #endif /* @@ -409,56 +422,167 @@ static const struct snd_soc_dai_ops hda_link_dai_ops = { struct snd_soc_dai_driver skl_dai[] = { { .name = "SSP0 Pin", + .playback = { + .channels_min = 1, + .channels_max = 8, + }, + .capture = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "SSP1 Pin", + .playback = { + .channels_min = 1, + .channels_max = 8, + }, + .capture = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "SSP2 Pin", + .playback = { + .channels_min = 1, + .channels_max = 8, + }, + .capture = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "SSP3 Pin", + .playback = { + .channels_min = 1, + .channels_max = 8, + }, + .capture = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "SSP4 Pin", + .playback = { + .channels_min = 1, + .channels_max = 8, + }, + .capture = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "SSP5 Pin", + .playback = { + .channels_min = 1, + .channels_max = 8, + }, + .capture = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "DMIC01 Pin", + .capture = { + .channels_min = 1, + .channels_max = 4, + }, }, { .name = "DMIC16k Pin", + .capture = { + .channels_min = 1, + .channels_max = 4, + }, }, #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) { .name = "iDisp1 Pin", .ops = &hda_link_dai_ops, + .playback = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "iDisp2 Pin", .ops = &hda_link_dai_ops, + .playback = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "iDisp3 Pin", .ops = &hda_link_dai_ops, + .playback = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "iDisp4 Pin", .ops = &hda_link_dai_ops, + .playback = { + .channels_min = 1, + .channels_max = 8, + }, }, { .name = "Analog CPU DAI", .ops = &hda_link_dai_ops, + .playback = { + .channels_min = 1, + .channels_max = 16, + }, + .capture = { + .channels_min = 1, + .channels_max = 16, + }, }, { .name = "Digital CPU DAI", .ops = &hda_link_dai_ops, + .playback = { + .channels_min = 1, + .channels_max = 16, + }, + .capture = { + .channels_min = 1, + .channels_max = 16, + }, }, { .name = "Alt Analog CPU DAI", .ops = &hda_link_dai_ops, + .playback = { + .channels_min = 1, + .channels_max = 16, + }, + .capture = { + .channels_min = 1, + .channels_max = 16, + }, +}, +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_PROBES) +{ + .name = "Probe Extraction CPU DAI", + .compress_new = snd_soc_new_compress, + .cops = &sof_probe_compr_ops, + .capture = { + .stream_name = "Probe Extraction", + .channels_min = 1, + .channels_max = 8, + .rates = SNDRV_PCM_RATE_48000, + .rate_min = 48000, + .rate_max = 48000, + }, }, #endif +#endif }; diff --git a/sound/soc/sof/intel/hda-dsp.c b/sound/soc/sof/intel/hda-dsp.c index 0848b79967a9..99087b6afb67 100644 --- a/sound/soc/sof/intel/hda-dsp.c +++ b/sound/soc/sof/intel/hda-dsp.c @@ -15,12 +15,21 @@ * Hardware interface for generic Intel audio DSP HDA IP */ +#include #include #include +#include "../sof-audio.h" #include "../ops.h" #include "hda.h" #include "hda-ipc.h" +static bool hda_enable_trace_D0I3_S0; +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG) +module_param_named(enable_trace_D0I3_S0, hda_enable_trace_D0I3_S0, bool, 0444); +MODULE_PARM_DESC(enable_trace_D0I3_S0, + "SOF HDA enable trace when the DSP is in D0I3 in S0"); +#endif + /* * DSP Core control. */ @@ -334,17 +343,15 @@ static int hda_dsp_send_pm_gate_ipc(struct snd_sof_dev *sdev, u32 flags) pm_gate.flags = flags; /* send pm_gate ipc to dsp */ - return sof_ipc_tx_message(sdev->ipc, pm_gate.hdr.cmd, &pm_gate, - sizeof(pm_gate), &reply, sizeof(reply)); + return sof_ipc_tx_message_no_pm(sdev->ipc, pm_gate.hdr.cmd, + &pm_gate, sizeof(pm_gate), &reply, + sizeof(reply)); } -int hda_dsp_set_power_state(struct snd_sof_dev *sdev, - enum sof_d0_substate d0_substate) +static int hda_dsp_update_d0i3c_register(struct snd_sof_dev *sdev, u8 value) { struct hdac_bus *bus = sof_to_bus(sdev); - u32 flags; int ret; - u8 value; /* Write to D0I3C after Command-In-Progress bit is cleared */ ret = hda_dsp_wait_d0i3c_done(sdev); @@ -354,7 +361,6 @@ int hda_dsp_set_power_state(struct snd_sof_dev *sdev, } /* Update D0I3C register */ - value = d0_substate == SOF_DSP_D0I3 ? SOF_HDA_VS_D0I3C_I3 : 0; snd_hdac_chip_updateb(bus, VS_D0I3C, SOF_HDA_VS_D0I3C_I3, value); /* Wait for cmd in progress to be cleared before exiting the function */ @@ -367,20 +373,218 @@ int hda_dsp_set_power_state(struct snd_sof_dev *sdev, dev_vdbg(bus->dev, "D0I3C updated, register = 0x%x\n", snd_hdac_chip_readb(bus, VS_D0I3C)); - if (d0_substate == SOF_DSP_D0I0) - flags = HDA_PM_PPG;/* prevent power gating in D0 */ - else - flags = HDA_PM_NO_DMA_TRACE;/* disable DMA trace in D0I3*/ + return 0; +} - /* sending pm_gate IPC */ - ret = hda_dsp_send_pm_gate_ipc(sdev, flags); +static int hda_dsp_set_D0_state(struct snd_sof_dev *sdev, + const struct sof_dsp_power_state *target_state) +{ + u32 flags = 0; + int ret; + u8 value = 0; + + /* + * Sanity check for illegal state transitions + * The only allowed transitions are: + * 1. D3 -> D0I0 + * 2. D0I0 -> D0I3 + * 3. D0I3 -> D0I0 + */ + switch (sdev->dsp_power_state.state) { + case SOF_DSP_PM_D0: + /* Follow the sequence below for D0 substate transitions */ + break; + case SOF_DSP_PM_D3: + /* Follow regular flow for D3 -> D0 transition */ + return 0; + default: + dev_err(sdev->dev, "error: transition from %d to %d not allowed\n", + sdev->dsp_power_state.state, target_state->state); + return -EINVAL; + } + + /* Set flags and register value for D0 target substate */ + if (target_state->substate == SOF_HDA_DSP_PM_D0I3) { + value = SOF_HDA_VS_D0I3C_I3; + + /* + * Trace DMA is disabled by default when the DSP enters D0I3. + * But it can be kept enabled when the DSP enters D0I3 while the + * system is in S0 for debug. + */ + if (hda_enable_trace_D0I3_S0 && + sdev->system_suspend_target != SOF_SUSPEND_NONE) + flags = HDA_PM_NO_DMA_TRACE; + } else { + /* prevent power gating in D0I0 */ + flags = HDA_PM_PPG; + } + + /* update D0I3C register */ + ret = hda_dsp_update_d0i3c_register(sdev, value); if (ret < 0) + return ret; + + /* + * Notify the DSP of the state change. + * If this IPC fails, revert the D0I3C register update in order + * to prevent partial state change. + */ + ret = hda_dsp_send_pm_gate_ipc(sdev, flags); + if (ret < 0) { dev_err(sdev->dev, "error: PM_GATE ipc error %d\n", ret); + goto revert; + } + + return ret; + +revert: + /* fallback to the previous register value */ + value = value ? 0 : SOF_HDA_VS_D0I3C_I3; + + /* + * This can fail but return the IPC error to signal that + * the state change failed. + */ + hda_dsp_update_d0i3c_register(sdev, value); return ret; } +/* helper to log DSP state */ +static void hda_dsp_state_log(struct snd_sof_dev *sdev) +{ + switch (sdev->dsp_power_state.state) { + case SOF_DSP_PM_D0: + switch (sdev->dsp_power_state.substate) { + case SOF_HDA_DSP_PM_D0I0: + dev_dbg(sdev->dev, "Current DSP power state: D0I0\n"); + break; + case SOF_HDA_DSP_PM_D0I3: + dev_dbg(sdev->dev, "Current DSP power state: D0I3\n"); + break; + default: + dev_dbg(sdev->dev, "Unknown DSP D0 substate: %d\n", + sdev->dsp_power_state.substate); + break; + } + break; + case SOF_DSP_PM_D1: + dev_dbg(sdev->dev, "Current DSP power state: D1\n"); + break; + case SOF_DSP_PM_D2: + dev_dbg(sdev->dev, "Current DSP power state: D2\n"); + break; + case SOF_DSP_PM_D3_HOT: + dev_dbg(sdev->dev, "Current DSP power state: D3_HOT\n"); + break; + case SOF_DSP_PM_D3: + dev_dbg(sdev->dev, "Current DSP power state: D3\n"); + break; + case SOF_DSP_PM_D3_COLD: + dev_dbg(sdev->dev, "Current DSP power state: D3_COLD\n"); + break; + default: + dev_dbg(sdev->dev, "Unknown DSP power state: %d\n", + sdev->dsp_power_state.state); + break; + } +} + +/* + * All DSP power state transitions are initiated by the driver. + * If the requested state change fails, the error is simply returned. + * Further state transitions are attempted only when the set_power_save() op + * is called again either because of a new IPC sent to the DSP or + * during system suspend/resume. + */ +int hda_dsp_set_power_state(struct snd_sof_dev *sdev, + const struct sof_dsp_power_state *target_state) +{ + int ret = 0; + + /* + * When the DSP is already in D0I3 and the target state is D0I3, + * it could be the case that the DSP is in D0I3 during S0 + * and the system is suspending to S0Ix. Therefore, + * hda_dsp_set_D0_state() must be called to disable trace DMA + * by sending the PM_GATE IPC to the FW. + */ + if (target_state->substate == SOF_HDA_DSP_PM_D0I3 && + sdev->system_suspend_target == SOF_SUSPEND_S0IX) + goto set_state; + + /* + * For all other cases, return without doing anything if + * the DSP is already in the target state. + */ + if (target_state->state == sdev->dsp_power_state.state && + target_state->substate == sdev->dsp_power_state.substate) + return 0; + +set_state: + switch (target_state->state) { + case SOF_DSP_PM_D0: + ret = hda_dsp_set_D0_state(sdev, target_state); + break; + case SOF_DSP_PM_D3: + /* The only allowed transition is: D0I0 -> D3 */ + if (sdev->dsp_power_state.state == SOF_DSP_PM_D0 && + sdev->dsp_power_state.substate == SOF_HDA_DSP_PM_D0I0) + break; + + dev_err(sdev->dev, + "error: transition from %d to %d not allowed\n", + sdev->dsp_power_state.state, target_state->state); + return -EINVAL; + default: + dev_err(sdev->dev, "error: target state unsupported %d\n", + target_state->state); + return -EINVAL; + } + if (ret < 0) { + dev_err(sdev->dev, + "failed to set requested target DSP state %d substate %d\n", + target_state->state, target_state->substate); + return ret; + } + + sdev->dsp_power_state = *target_state; + hda_dsp_state_log(sdev); + return ret; +} + +/* + * Audio DSP states may transform as below:- + * + * Opportunistic D0I3 in S0 + * Runtime +---------------------+ Delayed D0i3 work timeout + * suspend | +--------------------+ + * +------------+ D0I0(active) | | + * | | <---------------+ | + * | +--------> | New IPC | | + * | |Runtime +--^--+---------^--+--+ (via mailbox) | | + * | |resume | | | | | | + * | | | | | | | | + * | | System| | | | | | + * | | resume| | S3/S0IX | | | | + * | | | | suspend | | S0IX | | + * | | | | | |suspend | | + * | | | | | | | | + * | | | | | | | | + * +-v---+-----------+--v-------+ | | +------+----v----+ + * | | | +-----------> | + * | D3 (suspended) | | | D0I3 | + * | | +--------------+ | + * | | System resume | | + * +----------------------------+ +----------------+ + * + * S0IX suspend: The DSP is in D0I3 if any D0I3-compatible streams + * ignored the suspend trigger. Otherwise the DSP + * is in D3. + */ + static int hda_suspend(struct snd_sof_dev *sdev, bool runtime_suspend) { struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; @@ -390,6 +594,8 @@ static int hda_suspend(struct snd_sof_dev *sdev, bool runtime_suspend) #endif int ret; + hda_sdw_int_enable(sdev, false); + /* disable IPC interrupts */ hda_dsp_ipc_int_disable(sdev); @@ -486,10 +692,24 @@ int hda_dsp_resume(struct snd_sof_dev *sdev) { struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; struct pci_dev *pci = to_pci_dev(sdev->dev); + const struct sof_dsp_power_state target_state = { + .state = SOF_DSP_PM_D0, + .substate = SOF_HDA_DSP_PM_D0I0, + }; + int ret; - if (sdev->s0_suspend) { + /* resume from D0I3 */ + if (sdev->dsp_power_state.state == SOF_DSP_PM_D0) { hda_codec_i915_display_power(sdev, true); + /* Set DSP power state */ + ret = snd_sof_dsp_set_power_state(sdev, &target_state); + if (ret < 0) { + dev_err(sdev->dev, "error: setting dsp state %d substate %d\n", + target_state.state, target_state.substate); + return ret; + } + /* restore L1SEN bit */ if (hda->l1_support_changed) snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, @@ -503,13 +723,26 @@ int hda_dsp_resume(struct snd_sof_dev *sdev) } /* init hda controller. DSP cores will be powered up during fw boot */ - return hda_resume(sdev, false); + ret = hda_resume(sdev, false); + if (ret < 0) + return ret; + + return snd_sof_dsp_set_power_state(sdev, &target_state); } int hda_dsp_runtime_resume(struct snd_sof_dev *sdev) { + const struct sof_dsp_power_state target_state = { + .state = SOF_DSP_PM_D0, + }; + int ret; + /* init hda controller. DSP cores will be powered up during fw boot */ - return hda_resume(sdev, true); + ret = hda_resume(sdev, true); + if (ret < 0) + return ret; + + return snd_sof_dsp_set_power_state(sdev, &target_state); } int hda_dsp_runtime_idle(struct snd_sof_dev *sdev) @@ -527,21 +760,47 @@ int hda_dsp_runtime_idle(struct snd_sof_dev *sdev) int hda_dsp_runtime_suspend(struct snd_sof_dev *sdev) { + const struct sof_dsp_power_state target_state = { + .state = SOF_DSP_PM_D3, + }; + int ret; + /* stop hda controller and power dsp off */ - return hda_suspend(sdev, true); + ret = hda_suspend(sdev, true); + if (ret < 0) + return ret; + + return snd_sof_dsp_set_power_state(sdev, &target_state); } -int hda_dsp_suspend(struct snd_sof_dev *sdev) +int hda_dsp_suspend(struct snd_sof_dev *sdev, u32 target_state) { struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; struct hdac_bus *bus = sof_to_bus(sdev); struct pci_dev *pci = to_pci_dev(sdev->dev); + const struct sof_dsp_power_state target_dsp_state = { + .state = target_state, + .substate = target_state == SOF_DSP_PM_D0 ? + SOF_HDA_DSP_PM_D0I3 : 0, + }; int ret; - if (sdev->s0_suspend) { + /* cancel any attempt for DSP D0I3 */ + cancel_delayed_work_sync(&hda->d0i3_work); + + if (target_state == SOF_DSP_PM_D0) { /* we can't keep a wakeref to display driver at suspend */ hda_codec_i915_display_power(sdev, false); + /* Set DSP power state */ + ret = snd_sof_dsp_set_power_state(sdev, &target_dsp_state); + if (ret < 0) { + dev_err(sdev->dev, "error: setting dsp state %d substate %d\n", + target_dsp_state.state, + target_dsp_state.substate); + return ret; + } + /* enable L1SEN to make sure the system can enter S0Ix */ hda->l1_support_changed = snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, @@ -562,7 +821,7 @@ int hda_dsp_suspend(struct snd_sof_dev *sdev) return ret; } - return 0; + return snd_sof_dsp_set_power_state(sdev, &target_dsp_state); } int hda_dsp_set_hw_params_upon_resume(struct snd_sof_dev *sdev) @@ -588,7 +847,7 @@ int hda_dsp_set_hw_params_upon_resume(struct snd_sof_dev *sdev) */ if (stream->link_substream) { rtd = snd_pcm_substream_chip(stream->link_substream); - name = rtd->codec_dai->component->name; + name = asoc_rtd_to_codec(rtd, 0)->component->name; link = snd_hdac_ext_bus_get_link(bus, name); if (!link) return -EINVAL; @@ -606,3 +865,33 @@ int hda_dsp_set_hw_params_upon_resume(struct snd_sof_dev *sdev) #endif return 0; } + +void hda_dsp_d0i3_work(struct work_struct *work) +{ + struct sof_intel_hda_dev *hdev = container_of(work, + struct sof_intel_hda_dev, + d0i3_work.work); + struct hdac_bus *bus = &hdev->hbus.core; + struct snd_sof_dev *sdev = dev_get_drvdata(bus->dev); + struct sof_dsp_power_state target_state; + int ret; + + target_state.state = SOF_DSP_PM_D0; + + /* DSP can enter D0I3 iff only D0I3-compatible streams are active */ + if (snd_sof_dsp_only_d0i3_compatible_stream_active(sdev)) + target_state.substate = SOF_HDA_DSP_PM_D0I3; + else + target_state.substate = SOF_HDA_DSP_PM_D0I0; + + /* remain in D0I0 */ + if (target_state.substate == SOF_HDA_DSP_PM_D0I0) + return; + + /* This can fail but error cannot be propagated */ + ret = snd_sof_dsp_set_power_state(sdev, &target_state); + if (ret < 0) + dev_err_ratelimited(sdev->dev, + "error: failed to set DSP state %d substate %d\n", + target_state.state, target_state.substate); +} diff --git a/sound/soc/sof/intel/hda-ipc.c b/sound/soc/sof/intel/hda-ipc.c index 1837f66e361f..6062bb6011fb 100644 --- a/sound/soc/sof/intel/hda-ipc.c +++ b/sound/soc/sof/intel/hda-ipc.c @@ -106,7 +106,9 @@ void hda_dsp_ipc_get_reply(struct snd_sof_dev *sdev) ret = reply.error; } else { /* reply correct size ? */ - if (reply.hdr.size != msg->reply_size) { + if (reply.hdr.size != msg->reply_size && + /* getter payload is never known upfront */ + !(reply.hdr.cmd & SOF_IPC_GLB_PROBE)) { dev_err(sdev->dev, "error: reply expected %zu got %u bytes\n", msg->reply_size, reply.hdr.size); ret = -EINVAL; @@ -123,12 +125,6 @@ void hda_dsp_ipc_get_reply(struct snd_sof_dev *sdev) } -static bool hda_dsp_ipc_is_sof(uint32_t msg) -{ - return (msg & (HDA_DSP_IPC_PURGE_FW | 0xf << 9)) != msg || - (msg & HDA_DSP_IPC_PURGE_FW) != HDA_DSP_IPC_PURGE_FW; -} - /* IPC handler thread */ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context) { @@ -174,17 +170,9 @@ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context) */ spin_lock_irq(&sdev->ipc_lock); - /* handle immediate reply from DSP core - ignore ROM messages */ - if (hda_dsp_ipc_is_sof(msg)) { - hda_dsp_ipc_get_reply(sdev); - snd_sof_ipc_reply(sdev, msg); - } - - /* wake up sleeper if we are loading code */ - if (sdev->code_loading) { - sdev->code_loading = 0; - wake_up(&sdev->waitq); - } + /* handle immediate reply from DSP core */ + hda_dsp_ipc_get_reply(sdev); + snd_sof_ipc_reply(sdev, msg); /* set the done bit */ hda_dsp_ipc_dsp_done(sdev); diff --git a/sound/soc/sof/intel/hda-loader.c b/sound/soc/sof/intel/hda-loader.c index 8852184a2569..e1550ccd0a49 100644 --- a/sound/soc/sof/intel/hda-loader.c +++ b/sound/soc/sof/intel/hda-loader.c @@ -131,6 +131,12 @@ static int cl_dsp_init(struct snd_sof_dev *sdev, const void *fwdata, goto err; } + /* set DONE bit to clear the reply IPC message */ + snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR, + chip->ipc_ack, + chip->ipc_ack_mask, + chip->ipc_ack_mask); + /* step 5: power down corex */ ret = hda_dsp_core_power_down(sdev, chip->cores_mask & ~(HDA_DSP_CORE_MASK(0))); @@ -173,9 +179,6 @@ static int cl_trigger(struct snd_sof_dev *sdev, /* code loader is special case that reuses stream ops */ switch (cmd) { case SNDRV_PCM_TRIGGER_START: - wait_event_timeout(sdev->waitq, !sdev->code_loading, - HDA_DSP_CL_TRIGGER_TIMEOUT); - snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL, 1 << hstream->index, 1 << hstream->index); @@ -343,6 +346,24 @@ int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev) goto cleanup; } + /* + * When a SoundWire link is in clock stop state, a Slave + * device may trigger in-band wakes for events such as jack + * insertion or acoustic event detection. This event will lead + * to a WAKEEN interrupt, handled by the PCI device and routed + * to PME if the PCI device is in D3. The resume function in + * audio PCI driver will be invoked by ACPI for PME event and + * initialize the device and process WAKEEN interrupt. + * + * The WAKEEN interrupt should be processed ASAP to prevent an + * interrupt flood, otherwise other interrupts, such IPC, + * cannot work normally. The WAKEEN is handled after the ROM + * is initialized successfully, which ensures power rails are + * enabled before accessing the SoundWire SHIM registers + */ + if (!sdev->first_boot) + hda_sdw_process_wakeen(sdev); + /* * at this point DSP ROM has been initialized and * should be ready for code loading and firmware boot @@ -396,6 +417,19 @@ int hda_dsp_pre_fw_run(struct snd_sof_dev *sdev) /* post fw run operations */ int hda_dsp_post_fw_run(struct snd_sof_dev *sdev) { + int ret; + + if (sdev->first_boot) { + ret = hda_sdw_startup(sdev); + if (ret < 0) { + dev_err(sdev->dev, + "error: could not startup SoundWire links\n"); + return ret; + } + } + + hda_sdw_int_enable(sdev, true); + /* re-enable clock gating and power gating */ return hda_dsp_ctrl_clock_power_gating(sdev, true); } diff --git a/sound/soc/sof/intel/hda-pcm.c b/sound/soc/sof/intel/hda-pcm.c index 23872f6e708d..a46a6baa1c3f 100644 --- a/sound/soc/sof/intel/hda-pcm.c +++ b/sound/soc/sof/intel/hda-pcm.c @@ -27,7 +27,7 @@ #define SDnFMT_BITS(x) ((x) << 4) #define SDnFMT_CHAN(x) ((x) << 0) -static inline u32 get_mult_div(struct snd_sof_dev *sdev, int rate) +u32 hda_dsp_get_mult_div(struct snd_sof_dev *sdev, int rate) { switch (rate) { case 8000: @@ -61,7 +61,7 @@ static inline u32 get_mult_div(struct snd_sof_dev *sdev, int rate) } }; -static inline u32 get_bits(struct snd_sof_dev *sdev, int sample_bits) +u32 hda_dsp_get_bits(struct snd_sof_dev *sdev, int sample_bits) { switch (sample_bits) { case 8: @@ -95,8 +95,8 @@ int hda_dsp_pcm_hw_params(struct snd_sof_dev *sdev, u32 size, rate, bits; size = params_buffer_bytes(params); - rate = get_mult_div(sdev, params_rate(params)); - bits = get_bits(sdev, params_width(params)); + rate = hda_dsp_get_mult_div(sdev, params_rate(params)); + bits = hda_dsp_get_bits(sdev, params_width(params)); hstream->substream = substream; diff --git a/sound/soc/sof/intel/hda-stream.c b/sound/soc/sof/intel/hda-stream.c index c0ab9bb2a797..5d386956906f 100644 --- a/sound/soc/sof/intel/hda-stream.c +++ b/sound/soc/sof/intel/hda-stream.c @@ -547,6 +547,8 @@ int hda_dsp_stream_hw_free(struct snd_sof_dev *sdev, SOF_HDA_REG_PP_PPCTL, mask, 0); spin_unlock_irq(&bus->reg_lock); + stream->substream = NULL; + return 0; } @@ -571,6 +573,22 @@ bool hda_dsp_check_stream_irq(struct snd_sof_dev *sdev) return ret; } +static void +hda_dsp_set_bytes_transferred(struct hdac_stream *hstream, u64 buffer_size) +{ + u64 prev_pos, pos, num_bytes; + + div64_u64_rem(hstream->curr_pos, buffer_size, &prev_pos); + pos = snd_hdac_stream_get_pos_posbuf(hstream); + + if (pos < prev_pos) + num_bytes = (buffer_size - prev_pos) + pos; + else + num_bytes = pos - prev_pos; + + hstream->curr_pos += num_bytes; +} + static bool hda_dsp_stream_check(struct hdac_bus *bus, u32 status) { struct sof_intel_hda_dev *sof_hda = bus_to_sof_hda(bus); @@ -588,14 +606,19 @@ static bool hda_dsp_stream_check(struct hdac_bus *bus, u32 status) snd_hdac_stream_writeb(s, SD_STS, sd_status); active = true; - if (!s->substream || + if ((!s->substream && !s->cstream) || !s->running || (sd_status & SOF_HDA_CL_DMA_SD_INT_COMPLETE) == 0) continue; /* Inform ALSA only in case not do that with IPC */ - if (sof_hda->no_ipc_position) + if (s->substream && sof_hda->no_ipc_position) { snd_sof_pcm_period_elapsed(s->substream); + } else if (s->cstream) { + hda_dsp_set_bytes_transferred(s, + s->cstream->runtime->buffer_size); + snd_compr_fragment_elapsed(s->cstream); + } } } diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index 25946a1c2822..211e91e79eae 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -18,10 +18,14 @@ #include #include +#include #include +#include +#include #include #include #include +#include "../sof-audio.h" #include "../ops.h" #include "hda.h" @@ -34,6 +38,235 @@ #define EXCEPT_MAX_HDR_SIZE 0x400 +#if IS_ENABLED(CONFIG_SND_SOC_SOF_INTEL_SOUNDWIRE) + +/* + * The default for SoundWire clock stop quirks is to power gate the IP + * and do a Bus Reset, this will need to be modified when the DSP + * needs to remain in D0i3 so that the Master does not lose context + * and enumeration is not required on clock restart + */ +static int sdw_clock_stop_quirks = SDW_INTEL_CLK_STOP_BUS_RESET; +module_param(sdw_clock_stop_quirks, int, 0444); +MODULE_PARM_DESC(sdw_clock_stop_quirks, "SOF SoundWire clock stop quirks"); + +static int sdw_params_stream(struct device *dev, + struct sdw_intel_stream_params_data *params_data) +{ + struct snd_sof_dev *sdev = dev_get_drvdata(dev); + struct snd_soc_dai *d = params_data->dai; + struct sof_ipc_dai_config config; + struct sof_ipc_reply reply; + int link_id = params_data->link_id; + int alh_stream_id = params_data->alh_stream_id; + int ret; + u32 size = sizeof(config); + + memset(&config, 0, size); + config.hdr.size = size; + config.hdr.cmd = SOF_IPC_GLB_DAI_MSG | SOF_IPC_DAI_CONFIG; + config.type = SOF_DAI_INTEL_ALH; + config.dai_index = (link_id << 8) | (d->id); + config.alh.stream_id = alh_stream_id; + + /* send message to DSP */ + ret = sof_ipc_tx_message(sdev->ipc, + config.hdr.cmd, &config, size, &reply, + sizeof(reply)); + if (ret < 0) { + dev_err(sdev->dev, + "error: failed to set DAI hw_params for link %d dai->id %d ALH %d\n", + link_id, d->id, alh_stream_id); + } + + return ret; +} + +static int sdw_free_stream(struct device *dev, + struct sdw_intel_stream_free_data *free_data) +{ + struct snd_sof_dev *sdev = dev_get_drvdata(dev); + struct snd_soc_dai *d = free_data->dai; + struct sof_ipc_dai_config config; + struct sof_ipc_reply reply; + int link_id = free_data->link_id; + int ret; + u32 size = sizeof(config); + + memset(&config, 0, size); + config.hdr.size = size; + config.hdr.cmd = SOF_IPC_GLB_DAI_MSG | SOF_IPC_DAI_CONFIG; + config.type = SOF_DAI_INTEL_ALH; + config.dai_index = (link_id << 8) | d->id; + config.alh.stream_id = 0xFFFF; /* invalid value on purpose */ + + /* send message to DSP */ + ret = sof_ipc_tx_message(sdev->ipc, + config.hdr.cmd, &config, size, &reply, + sizeof(reply)); + if (ret < 0) { + dev_err(sdev->dev, + "error: failed to free stream for link %d dai->id %d\n", + link_id, d->id); + } + + return ret; +} + +static const struct sdw_intel_ops sdw_callback = { + .params_stream = sdw_params_stream, + .free_stream = sdw_free_stream, +}; + +void hda_sdw_int_enable(struct snd_sof_dev *sdev, bool enable) +{ + sdw_intel_enable_irq(sdev->bar[HDA_DSP_BAR], enable); +} + +static int hda_sdw_acpi_scan(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hdev; + acpi_handle handle; + int ret; + + handle = ACPI_HANDLE(sdev->dev); + + /* save ACPI info for the probe step */ + hdev = sdev->pdata->hw_pdata; + + ret = sdw_intel_acpi_scan(handle, &hdev->info); + if (ret < 0) { + dev_err(sdev->dev, "%s failed\n", __func__); + return -EINVAL; + } + + return 0; +} + +static int hda_sdw_probe(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hdev; + struct sdw_intel_res res; + void *sdw; + + hdev = sdev->pdata->hw_pdata; + + memset(&res, 0, sizeof(res)); + + res.mmio_base = sdev->bar[HDA_DSP_BAR]; + res.irq = sdev->ipc_irq; + res.handle = hdev->info.handle; + res.parent = sdev->dev; + res.ops = &sdw_callback; + res.dev = sdev->dev; + res.clock_stop_quirks = sdw_clock_stop_quirks; + + /* + * ops and arg fields are not populated for now, + * they will be needed when the DAI callbacks are + * provided + */ + + /* we could filter links here if needed, e.g for quirks */ + res.count = hdev->info.count; + res.link_mask = hdev->info.link_mask; + + sdw = sdw_intel_probe(&res); + if (!sdw) { + dev_err(sdev->dev, "error: SoundWire probe failed\n"); + return -EINVAL; + } + + /* save context */ + hdev->sdw = sdw; + + return 0; +} + +int hda_sdw_startup(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hdev; + + hdev = sdev->pdata->hw_pdata; + + if (!hdev->sdw) + return 0; + + return sdw_intel_startup(hdev->sdw); +} + +static int hda_sdw_exit(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hdev; + + hdev = sdev->pdata->hw_pdata; + + hda_sdw_int_enable(sdev, false); + + if (hdev->sdw) + sdw_intel_exit(hdev->sdw); + hdev->sdw = NULL; + + return 0; +} + +static bool hda_dsp_check_sdw_irq(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hdev; + bool ret = false; + u32 irq_status; + + hdev = sdev->pdata->hw_pdata; + + if (!hdev->sdw) + return ret; + + /* store status */ + irq_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIS2); + + /* invalid message ? */ + if (irq_status == 0xffffffff) + goto out; + + /* SDW message ? */ + if (irq_status & HDA_DSP_REG_ADSPIS2_SNDW) + ret = true; + +out: + return ret; +} + +static irqreturn_t hda_dsp_sdw_thread(int irq, void *context) +{ + return sdw_intel_thread(irq, context); +} + +static bool hda_sdw_check_wakeen_irq(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hdev; + + hdev = sdev->pdata->hw_pdata; + if (hdev->sdw && + snd_sof_dsp_read(sdev, HDA_DSP_BAR, + HDA_DSP_REG_SNDW_WAKE_STS)) + return true; + + return false; +} + +void hda_sdw_process_wakeen(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hdev; + + hdev = sdev->pdata->hw_pdata; + if (!hdev->sdw) + return; + + sdw_intel_process_wakeen_event(hdev->sdw); +} + +#endif + /* * Debug */ @@ -54,8 +287,7 @@ static int hda_dmic_num = -1; module_param_named(dmic_num, hda_dmic_num, int, 0444); MODULE_PARM_DESC(dmic_num, "SOF HDA DMIC number"); -static bool hda_codec_use_common_hdmi = - IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_COMMON_HDMI_CODEC); +static bool hda_codec_use_common_hdmi = IS_ENABLED(CONFIG_SND_HDA_CODEC_HDMI); module_param_named(use_common_hdmi, hda_codec_use_common_hdmi, bool, 0444); MODULE_PARM_DESC(use_common_hdmi, "SOF HDA use common HDMI codec driver"); #endif @@ -288,10 +520,8 @@ static int hda_init(struct snd_sof_dev *sdev) /* init i915 and HDMI codecs */ ret = hda_codec_i915_init(sdev); - if (ret < 0) { - dev_err(sdev->dev, "error: init i915 and HDMI codec failed\n"); - return ret; - } + if (ret < 0) + dev_warn(sdev->dev, "init of i915 and HDMI codec failed\n"); /* get controller capabilities */ ret = hda_dsp_ctrl_get_caps(sdev); @@ -349,9 +579,12 @@ static const char *fixup_tplg_name(struct snd_sof_dev *sdev, static int hda_init_caps(struct snd_sof_dev *sdev) { struct hdac_bus *bus = sof_to_bus(sdev); + struct snd_sof_pdata *pdata = sdev->pdata; #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) struct hdac_ext_link *hlink; #endif + struct sof_intel_hda_dev *hdev = pdata->hw_pdata; + u32 link_mask; int ret = 0; device_disable_async_suspend(bus->dev); @@ -365,12 +598,37 @@ static int hda_init_caps(struct snd_sof_dev *sdev) if (ret < 0) { dev_err(bus->dev, "error: init chip failed with ret: %d\n", ret); -#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) - hda_codec_i915_exit(sdev); -#endif return ret; } + /* scan SoundWire capabilities exposed by DSDT */ + ret = hda_sdw_acpi_scan(sdev); + if (ret < 0) { + dev_dbg(sdev->dev, "skipping SoundWire, ACPI scan error\n"); + goto skip_soundwire; + } + + link_mask = hdev->info.link_mask; + if (!link_mask) { + dev_dbg(sdev->dev, "skipping SoundWire, no links enabled\n"); + goto skip_soundwire; + } + + /* + * probe/allocate SoundWire resources. + * The hardware configuration takes place in hda_sdw_startup + * after power rails are enabled. + * It's entirely possible to have a mix of I2S/DMIC/SoundWire + * devices, so we allocate the resources in all cases. + */ + ret = hda_sdw_probe(sdev); + if (ret < 0) { + dev_err(sdev->dev, "error: SoundWire probe error\n"); + return ret; + } + +skip_soundwire: + #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) if (bus->mlcap) snd_hdac_ext_bus_get_ml_capabilities(bus); @@ -379,7 +637,7 @@ static int hda_init_caps(struct snd_sof_dev *sdev) hda_codec_probe_bus(sdev, hda_codec_use_common_hdmi); if (!HDA_IDISP_CODEC(bus->codec_mask)) - hda_codec_i915_exit(sdev); + hda_codec_i915_display_power(sdev, false); /* * we are done probing so decrement link counts @@ -427,6 +685,7 @@ static irqreturn_t hda_dsp_interrupt_handler(int irq, void *context) static irqreturn_t hda_dsp_interrupt_thread(int irq, void *context) { struct snd_sof_dev *sdev = context; + struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; /* deal with streams and controller first */ if (hda_dsp_check_stream_irq(sdev)) @@ -435,6 +694,12 @@ static irqreturn_t hda_dsp_interrupt_thread(int irq, void *context) if (hda_dsp_check_ipc_irq(sdev)) sof_ops(sdev)->irq_thread(irq, sdev); + if (hda_dsp_check_sdw_irq(sdev)) + hda_dsp_sdw_thread(irq, hdev->sdw); + + if (hda_sdw_check_wakeen_irq(sdev)) + hda_sdw_process_wakeen(sdev); + /* enable GIE interrupt */ snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL, @@ -590,12 +855,11 @@ int hda_dsp_probe(struct snd_sof_dev *sdev) hda_dsp_ctrl_ppcap_enable(sdev, true); hda_dsp_ctrl_ppcap_int_enable(sdev, true); - /* initialize waitq for code loading */ - init_waitqueue_head(&sdev->waitq); - /* set default mailbox offset for FW ready message */ sdev->dsp_box.offset = HDA_DSP_MBOX_UPLINK_OFFSET; + INIT_DELAYED_WORK(&hdev->d0i3_work, hda_dsp_d0i3_work); + return 0; free_ipc_irq: @@ -621,11 +885,16 @@ int hda_dsp_remove(struct snd_sof_dev *sdev) struct pci_dev *pci = to_pci_dev(sdev->dev); const struct sof_intel_dsp_desc *chip = hda->desc; + /* cancel any attempt for DSP D0I3 */ + cancel_delayed_work_sync(&hda->d0i3_work); + #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) /* codec removal, invoke bus_device_remove */ snd_hdac_ext_bus_device_remove(bus); #endif + hda_sdw_exit(sdev); + if (!IS_ERR_OR_NULL(hda->dmic_dev)) platform_device_unregister(hda->dmic_dev); @@ -694,12 +963,11 @@ static int hda_generic_machine_select(struct snd_sof_dev *sdev) /* * If no machine driver is found, then: * - * hda machine driver is used if : - * 1. there is one HDMI codec and one external HDAudio codec - * 2. only HDMI codec + * generic hda machine driver can handle: + * - one HDMI codec, and/or + * - one external HDAudio codec */ - if (!pdata->machine && codec_num <= 2 && - HDA_IDISP_CODEC(bus->codec_mask)) { + if (!pdata->machine && codec_num <= 2) { hda_mach = snd_soc_acpi_intel_hda_machines; /* topology: use the info from hda_machines */ @@ -709,7 +977,7 @@ static int hda_generic_machine_select(struct snd_sof_dev *sdev) dev_info(bus->dev, "using HDA machine driver %s now\n", hda_mach->drv_name); - if (codec_num == 1) + if (codec_num == 1 && HDA_IDISP_CODEC(bus->codec_mask)) idisp_str = "-idisp"; else idisp_str = ""; @@ -763,6 +1031,123 @@ static int hda_generic_machine_select(struct snd_sof_dev *sdev) } #endif +#if IS_ENABLED(CONFIG_SND_SOC_SOF_INTEL_SOUNDWIRE) +/* Check if all Slaves defined on the link can be found */ +static bool link_slaves_found(struct snd_sof_dev *sdev, + const struct snd_soc_acpi_link_adr *link, + struct sdw_intel_ctx *sdw) +{ + struct hdac_bus *bus = sof_to_bus(sdev); + struct sdw_intel_slave_id *ids = sdw->ids; + int num_slaves = sdw->num_slaves; + unsigned int part_id, link_id, unique_id, mfg_id; + int i, j; + + for (i = 0; i < link->num_adr; i++) { + u64 adr = link->adr_d[i].adr; + + mfg_id = SDW_MFG_ID(adr); + part_id = SDW_PART_ID(adr); + link_id = SDW_DISCO_LINK_ID(adr); + for (j = 0; j < num_slaves; j++) { + if (ids[j].link_id != link_id || + ids[j].id.part_id != part_id || + ids[j].id.mfg_id != mfg_id) + continue; + /* + * we have to check unique id + * if there is more than one + * Slave on the link + */ + unique_id = SDW_UNIQUE_ID(adr); + if (link->num_adr == 1 || + ids[j].id.unique_id == SDW_IGNORED_UNIQUE_ID || + ids[j].id.unique_id == unique_id) { + dev_dbg(bus->dev, + "found %x at link %d\n", + part_id, link_id); + break; + } + } + if (j == num_slaves) { + dev_dbg(bus->dev, + "Slave %x not found\n", + part_id); + return false; + } + } + return true; +} + +static int hda_sdw_machine_select(struct snd_sof_dev *sdev) +{ + struct snd_sof_pdata *pdata = sdev->pdata; + const struct snd_soc_acpi_link_adr *link; + struct hdac_bus *bus = sof_to_bus(sdev); + struct snd_soc_acpi_mach *mach; + struct sof_intel_hda_dev *hdev; + u32 link_mask; + int i; + + hdev = pdata->hw_pdata; + link_mask = hdev->info.link_mask; + + /* + * Select SoundWire machine driver if needed using the + * alternate tables. This case deals with SoundWire-only + * machines, for mixed cases with I2C/I2S the detection relies + * on the HID list. + */ + if (link_mask && !pdata->machine) { + for (mach = pdata->desc->alt_machines; + mach && mach->link_mask; mach++) { + if (mach->link_mask != link_mask) + continue; + + /* No need to match adr if there is no links defined */ + if (!mach->links) + break; + + link = mach->links; + for (i = 0; i < hdev->info.count && link->num_adr; + i++, link++) { + /* + * Try next machine if any expected Slaves + * are not found on this link. + */ + if (!link_slaves_found(sdev, link, hdev->sdw)) + break; + } + /* Found if all Slaves are checked */ + if (i == hdev->info.count || !link->num_adr) + break; + } + if (mach && mach->link_mask) { + dev_dbg(bus->dev, + "SoundWire machine driver %s topology %s\n", + mach->drv_name, + mach->sof_tplg_filename); + pdata->machine = mach; + mach->mach_params.links = mach->links; + mach->mach_params.link_mask = mach->link_mask; + mach->mach_params.platform = dev_name(sdev->dev); + pdata->fw_filename = mach->sof_fw_filename; + pdata->tplg_filename = mach->sof_tplg_filename; + } else { + dev_info(sdev->dev, + "No SoundWire machine driver found\n"); + } + } + + return 0; +} +#else +static int hda_sdw_machine_select(struct snd_sof_dev *sdev) +{ + return 0; +} +#endif + void hda_set_mach_params(const struct snd_soc_acpi_mach *mach, struct device *dev) { @@ -782,8 +1167,18 @@ void hda_machine_select(struct snd_sof_dev *sdev) if (mach) { sof_pdata->tplg_filename = mach->sof_tplg_filename; sof_pdata->machine = mach; + + if (mach->link_mask) { + mach->mach_params.links = mach->links; + mach->mach_params.link_mask = mach->link_mask; + } } + /* + * If I2S fails, try SoundWire + */ + hda_sdw_machine_select(sdev); + /* * Choose HDA generic machine driver if mach is NULL. * Otherwise, set certain mach params. diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h index 6191d9192fae..e9825798de77 100644 --- a/sound/soc/sof/intel/hda.h +++ b/sound/soc/sof/intel/hda.h @@ -11,6 +11,9 @@ #ifndef __SOF_INTEL_HDA_H #define __SOF_INTEL_HDA_H +#include +#include +#include #include #include #include "shim.h" @@ -174,7 +177,6 @@ * value cannot be read back within the specified time. */ #define HDA_DSP_STREAM_RUN_TIMEOUT 300 -#define HDA_DSP_CL_TRIGGER_TIMEOUT 300 #define HDA_DSP_SPIB_ENABLE 1 #define HDA_DSP_SPIB_DISABLE 0 @@ -230,6 +232,9 @@ #define HDA_DSP_REG_ADSPIC2 (HDA_DSP_GEN_BASE + 0x10) #define HDA_DSP_REG_ADSPIS2 (HDA_DSP_GEN_BASE + 0x14) +#define HDA_DSP_REG_ADSPIS2_SNDW BIT(5) +#define HDA_DSP_REG_SNDW_WAKE_STS 0x2C192 + /* Intel HD Audio Inter-Processor Communication Registers */ #define HDA_DSP_IPC_BASE 0x40 #define HDA_DSP_REG_HIPCT (HDA_DSP_IPC_BASE + 0x00) @@ -348,7 +353,13 @@ /* Number of DAIs */ #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_PROBES) +#define SOF_SKL_NUM_DAIS 16 +#else #define SOF_SKL_NUM_DAIS 15 +#endif + #else #define SOF_SKL_NUM_DAIS 8 #endif @@ -392,6 +403,19 @@ struct sof_intel_dsp_bdl { #define SOF_HDA_PLAYBACK 0 #define SOF_HDA_CAPTURE 1 +/* + * Time in ms for opportunistic D0I3 entry delay. + * This has been deliberately chosen to be long to avoid race conditions. + * Could be optimized in future. + */ +#define SOF_HDA_D0I3_WORK_DELAY_MS 5000 + +/* HDA DSP D0 substate */ +enum sof_hda_D0_substate { + SOF_HDA_DSP_PM_D0I0, /* default D0 substate */ + SOF_HDA_DSP_PM_D0I3, /* low power D0 substate */ +}; + /* represents DSP HDA controller frontend - i.e. host facing control */ struct sof_intel_hda_dev { @@ -414,6 +438,15 @@ struct sof_intel_hda_dev { /* DMIC device */ struct platform_device *dmic_dev; + + /* delayed work to enter D0I3 opportunistically */ + struct delayed_work d0i3_work; + + /* ACPI information stored between scan and probe steps */ + struct sdw_intel_acpi_info info; + + /* sdw context allocated by SoundWire driver */ + struct sdw_intel_ctx *sdw; }; static inline struct hdac_bus *sof_to_bus(struct snd_sof_dev *s) @@ -469,9 +502,9 @@ void hda_dsp_ipc_int_enable(struct snd_sof_dev *sdev); void hda_dsp_ipc_int_disable(struct snd_sof_dev *sdev); int hda_dsp_set_power_state(struct snd_sof_dev *sdev, - enum sof_d0_substate d0_substate); + const struct sof_dsp_power_state *target_state); -int hda_dsp_suspend(struct snd_sof_dev *sdev); +int hda_dsp_suspend(struct snd_sof_dev *sdev, u32 target_state); int hda_dsp_resume(struct snd_sof_dev *sdev); int hda_dsp_runtime_suspend(struct snd_sof_dev *sdev); int hda_dsp_runtime_resume(struct snd_sof_dev *sdev); @@ -481,10 +514,13 @@ void hda_dsp_dump_skl(struct snd_sof_dev *sdev, u32 flags); void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags); void hda_ipc_dump(struct snd_sof_dev *sdev); void hda_ipc_irq_dump(struct snd_sof_dev *sdev); +void hda_dsp_d0i3_work(struct work_struct *work); /* * DSP PCM Operations. */ +u32 hda_dsp_get_mult_div(struct snd_sof_dev *sdev, int rate); +u32 hda_dsp_get_bits(struct snd_sof_dev *sdev, int sample_bits); int hda_dsp_pcm_open(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream); int hda_dsp_pcm_close(struct snd_sof_dev *sdev, @@ -533,6 +569,29 @@ int hda_ipc_pcm_params(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream, const struct sof_ipc_pcm_params_reply *reply); +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_PROBES) +/* + * Probe Compress Operations. + */ +int hda_probe_compr_assign(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_soc_dai *dai); +int hda_probe_compr_free(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_soc_dai *dai); +int hda_probe_compr_set_params(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_compr_params *params, + struct snd_soc_dai *dai); +int hda_probe_compr_trigger(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, int cmd, + struct snd_soc_dai *dai); +int hda_probe_compr_pointer(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_compr_tstamp *tstamp, + struct snd_soc_dai *dai); +#endif + /* * DSP IPC Operations. */ @@ -606,6 +665,61 @@ int hda_dsp_trace_init(struct snd_sof_dev *sdev, u32 *stream_tag); int hda_dsp_trace_release(struct snd_sof_dev *sdev); int hda_dsp_trace_trigger(struct snd_sof_dev *sdev, int cmd); +/* + * SoundWire support + */ +#if IS_ENABLED(CONFIG_SND_SOC_SOF_INTEL_SOUNDWIRE) + +int hda_sdw_startup(struct snd_sof_dev *sdev); +void hda_sdw_int_enable(struct snd_sof_dev *sdev, bool enable); +void hda_sdw_process_wakeen(struct snd_sof_dev *sdev); + +#else + +static inline int hda_sdw_acpi_scan(struct snd_sof_dev *sdev) +{ + return 0; +} + +static inline int hda_sdw_probe(struct snd_sof_dev *sdev) +{ + return 0; +} + +static inline int hda_sdw_startup(struct snd_sof_dev *sdev) +{ + return 0; +} + +static inline int hda_sdw_exit(struct snd_sof_dev *sdev) +{ + return 0; +} + +static inline void hda_sdw_int_enable(struct snd_sof_dev *sdev, bool enable) +{ +} + +static inline bool hda_dsp_check_sdw_irq(struct snd_sof_dev *sdev) +{ + return false; +} + +static inline irqreturn_t hda_dsp_sdw_thread(int irq, void *context) +{ + return IRQ_HANDLED; +} + +static inline bool hda_sdw_check_wakeen_irq(struct snd_sof_dev *sdev) +{ + return false; +} + +static inline void hda_sdw_process_wakeen(struct snd_sof_dev *sdev) +{ +} +#endif + /* common dai driver */ extern struct snd_soc_dai_driver skl_dai[]; diff --git a/sound/soc/sof/ipc.c b/sound/soc/sof/ipc.c index 78aa1da7c7a9..1c6794918cbb 100644 --- a/sound/soc/sof/ipc.c +++ b/sound/soc/sof/ipc.c @@ -214,15 +214,17 @@ static int tx_wait_done(struct snd_sof_ipc *ipc, struct snd_sof_ipc_msg *msg, snd_sof_handle_fw_exception(ipc->sdev); ret = -ETIMEDOUT; } else { - /* copy the data returned from DSP */ ret = msg->reply_error; - if (msg->reply_size) - memcpy(reply_data, msg->reply_data, msg->reply_size); - if (ret < 0) + if (ret < 0) { dev_err(sdev->dev, "error: ipc error for 0x%x size %zu\n", hdr->cmd, msg->reply_size); - else + } else { ipc_log_header(sdev->dev, "ipc tx succeeded", hdr->cmd); + if (msg->reply_size) + /* copy the data returned from DSP */ + memcpy(reply_data, msg->reply_data, + msg->reply_size); + } } return ret; @@ -268,7 +270,6 @@ static int sof_ipc_tx_message_unlocked(struct snd_sof_ipc *ipc, u32 header, spin_unlock_irq(&sdev->ipc_lock); if (ret < 0) { - /* So far IPC TX never fails, consider making the above void */ dev_err_ratelimited(sdev->dev, "error: ipc tx failed with error %d\n", ret); @@ -288,6 +289,32 @@ static int sof_ipc_tx_message_unlocked(struct snd_sof_ipc *ipc, u32 header, int sof_ipc_tx_message(struct snd_sof_ipc *ipc, u32 header, void *msg_data, size_t msg_bytes, void *reply_data, size_t reply_bytes) +{ + const struct sof_dsp_power_state target_state = { + .state = SOF_DSP_PM_D0, + }; + int ret; + + /* ensure the DSP is in D0 before sending a new IPC */ + ret = snd_sof_dsp_set_power_state(ipc->sdev, &target_state); + if (ret < 0) { + dev_err(ipc->sdev->dev, "error: resuming DSP %d\n", ret); + return ret; + } + + return sof_ipc_tx_message_no_pm(ipc, header, msg_data, msg_bytes, + reply_data, reply_bytes); +} +EXPORT_SYMBOL(sof_ipc_tx_message); + +/* + * send IPC message from host to DSP without modifying the DSP state. + * This will be used for IPC's that can be handled by the DSP + * even in a low-power D0 substate. + */ +int sof_ipc_tx_message_no_pm(struct snd_sof_ipc *ipc, u32 header, + void *msg_data, size_t msg_bytes, + void *reply_data, size_t reply_bytes) { int ret; @@ -305,7 +332,7 @@ int sof_ipc_tx_message(struct snd_sof_ipc *ipc, u32 header, return ret; } -EXPORT_SYMBOL(sof_ipc_tx_message); +EXPORT_SYMBOL(sof_ipc_tx_message_no_pm); /* handle reply message from DSP */ int snd_sof_ipc_reply(struct snd_sof_dev *sdev, u32 msg_id) diff --git a/sound/soc/sof/loader.c b/sound/soc/sof/loader.c index fc4ab51bacf4..1f2e0be812bd 100644 --- a/sound/soc/sof/loader.c +++ b/sound/soc/sof/loader.c @@ -95,9 +95,6 @@ int snd_sof_fw_parse_ext_data(struct snd_sof_dev *sdev, u32 bar, u32 offset) /* process structure data */ switch (ext_hdr->type) { - case SOF_IPC_EXT_DMA_BUFFER: - ret = 0; - break; case SOF_IPC_EXT_WINDOW: ret = get_ext_windows(sdev, ext_hdr); break; @@ -469,9 +466,6 @@ int snd_sof_load_firmware_raw(struct snd_sof_dev *sdev) const char *fw_filename; int ret; - /* set code loading condition to true */ - sdev->code_loading = 1; - /* Don't request firmware again if firmware is already requested */ if (plat_data->fw) return 0; diff --git a/sound/soc/sof/ops.h b/sound/soc/sof/ops.h index e929a6e0058f..a771500ac442 100644 --- a/sound/soc/sof/ops.h +++ b/sound/soc/sof/ops.h @@ -146,10 +146,11 @@ static inline int snd_sof_dsp_resume(struct snd_sof_dev *sdev) return 0; } -static inline int snd_sof_dsp_suspend(struct snd_sof_dev *sdev) +static inline int snd_sof_dsp_suspend(struct snd_sof_dev *sdev, + u32 target_state) { if (sof_ops(sdev)->suspend) - return sof_ops(sdev)->suspend(sdev); + return sof_ops(sdev)->suspend(sdev, target_state); return 0; } @@ -193,14 +194,15 @@ static inline int snd_sof_dsp_set_clk(struct snd_sof_dev *sdev, u32 freq) return 0; } -static inline int snd_sof_dsp_set_power_state(struct snd_sof_dev *sdev, - enum sof_d0_substate substate) +static inline int +snd_sof_dsp_set_power_state(struct snd_sof_dev *sdev, + const struct sof_dsp_power_state *target_state) { if (sof_ops(sdev)->set_power_state) - return sof_ops(sdev)->set_power_state(sdev, substate); + return sof_ops(sdev)->set_power_state(sdev, target_state); - /* D0 substate is not supported */ - return -ENOTSUPP; + /* D0 substate is not supported, do nothing here. */ + return 0; } /* debug */ @@ -391,6 +393,49 @@ snd_sof_pcm_platform_pointer(struct snd_sof_dev *sdev, return 0; } +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_PROBES) +static inline int +snd_sof_probe_compr_assign(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, struct snd_soc_dai *dai) +{ + return sof_ops(sdev)->probe_assign(sdev, cstream, dai); +} + +static inline int +snd_sof_probe_compr_free(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, struct snd_soc_dai *dai) +{ + return sof_ops(sdev)->probe_free(sdev, cstream, dai); +} + +static inline int +snd_sof_probe_compr_set_params(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_compr_params *params, struct snd_soc_dai *dai) +{ + return sof_ops(sdev)->probe_set_params(sdev, cstream, params, dai); +} + +static inline int +snd_sof_probe_compr_trigger(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, int cmd, + struct snd_soc_dai *dai) +{ + return sof_ops(sdev)->probe_trigger(sdev, cstream, cmd, dai); +} + +static inline int +snd_sof_probe_compr_pointer(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_compr_tstamp *tstamp, struct snd_soc_dai *dai) +{ + if (sof_ops(sdev) && sof_ops(sdev)->probe_pointer) + return sof_ops(sdev)->probe_pointer(sdev, cstream, tstamp, dai); + + return 0; +} +#endif + /* machine driver */ static inline int snd_sof_machine_register(struct snd_sof_dev *sdev, void *pdata) diff --git a/sound/soc/sof/pcm.c b/sound/soc/sof/pcm.c index 29435ba2d329..47cd741f2a8c 100644 --- a/sound/soc/sof/pcm.c +++ b/sound/soc/sof/pcm.c @@ -16,6 +16,9 @@ #include "sof-priv.h" #include "sof-audio.h" #include "ops.h" +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_PROBES) +#include "compress.h" +#endif /* Create DMA buffer page table for DSP */ static int create_page_table(struct snd_soc_component *component, @@ -54,7 +57,7 @@ static int sof_pcm_dsp_params(struct snd_sof_pcm *spcm, struct snd_pcm_substream /* * sof pcm period elapse work */ -static void sof_pcm_period_elapsed_work(struct work_struct *work) +void snd_sof_pcm_period_elapsed_work(struct work_struct *work) { struct snd_sof_pcm_stream *sps = container_of(work, struct snd_sof_pcm_stream, @@ -372,7 +375,7 @@ static int sof_pcm_trigger(struct snd_soc_component *component, stream.hdr.cmd |= SOF_IPC_STREAM_TRIG_START; break; case SNDRV_PCM_TRIGGER_SUSPEND: - if (sdev->s0_suspend && + if (sdev->system_suspend_target == SOF_SUSPEND_S0IX && spcm->stream[substream->stream].d0i3_compatible) { /* * trap the event, not sending trigger stop to @@ -472,8 +475,6 @@ static int sof_pcm_open(struct snd_soc_component *component, dev_dbg(component->dev, "pcm: open stream %d dir %d\n", spcm->pcm.pcm_id, substream->stream); - INIT_WORK(&spcm->stream[substream->stream].period_elapsed_work, - sof_pcm_period_elapsed_work); caps = &spcm->pcm.caps[substream->stream]; @@ -598,8 +599,7 @@ static int sof_pcm_new(struct snd_soc_component *component, snd_pcm_set_managed_buffer(pcm->streams[stream].substream, SNDRV_DMA_TYPE_DEV_SG, sdev->dev, - le32_to_cpu(caps->buffer_size_min), - le32_to_cpu(caps->buffer_size_max)); + 0, le32_to_cpu(caps->buffer_size_max)); capture: stream = SNDRV_PCM_STREAM_CAPTURE; @@ -621,8 +621,7 @@ static int sof_pcm_new(struct snd_soc_component *component, snd_pcm_set_managed_buffer(pcm->streams[stream].substream, SNDRV_DMA_TYPE_DEV_SG, sdev->dev, - le32_to_cpu(caps->buffer_size_min), - le32_to_cpu(caps->buffer_size_max)); + 0, le32_to_cpu(caps->buffer_size_max)); return 0; } @@ -787,6 +786,10 @@ void snd_sof_new_platform_drv(struct snd_sof_dev *sdev) #if IS_ENABLED(CONFIG_SND_SOC_SOF_COMPRESS) pd->compr_ops = &sof_compressed_ops; +#endif +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_PROBES) + /* override cops when probe support is enabled */ + pd->compr_ops = &sof_probe_compressed_ops; #endif pd->pcm_construct = sof_pcm_new; pd->ignore_machine = drv_name; diff --git a/sound/soc/sof/pm.c b/sound/soc/sof/pm.c index a0cde053b61a..c410822d9920 100644 --- a/sound/soc/sof/pm.c +++ b/sound/soc/sof/pm.c @@ -12,6 +12,42 @@ #include "sof-priv.h" #include "sof-audio.h" +/* + * Helper function to determine the target DSP state during + * system suspend. This function only cares about the device + * D-states. Platform-specific substates, if any, should be + * handled by the platform-specific parts. + */ +static u32 snd_sof_dsp_power_target(struct snd_sof_dev *sdev) +{ + u32 target_dsp_state; + + switch (sdev->system_suspend_target) { + case SOF_SUSPEND_S3: + /* DSP should be in D3 if the system is suspending to S3 */ + target_dsp_state = SOF_DSP_PM_D3; + break; + case SOF_SUSPEND_S0IX: + /* + * Currently, the only criterion for retaining the DSP in D0 + * is that there are streams that ignored the suspend trigger. + * Additional criteria such Soundwire clock-stop mode and + * device suspend latency considerations will be added later. + */ + if (snd_sof_stream_suspend_ignored(sdev)) + target_dsp_state = SOF_DSP_PM_D0; + else + target_dsp_state = SOF_DSP_PM_D3; + break; + default: + /* This case would be during runtime suspend */ + target_dsp_state = SOF_DSP_PM_D3; + break; + } + + return target_dsp_state; +} + static int sof_send_pm_ctx_ipc(struct snd_sof_dev *sdev, int cmd) { struct sof_ipc_pm_ctx pm_ctx; @@ -50,6 +86,7 @@ static void sof_cache_debugfs(struct snd_sof_dev *sdev) static int sof_resume(struct device *dev, bool runtime_resume) { struct snd_sof_dev *sdev = dev_get_drvdata(dev); + u32 old_state = sdev->dsp_power_state.state; int ret; /* do nothing if dsp resume callbacks are not set */ @@ -74,6 +111,10 @@ static int sof_resume(struct device *dev, bool runtime_resume) return ret; } + /* Nothing further to do if resuming from a low-power D0 substate */ + if (!runtime_resume && old_state == SOF_DSP_PM_D0) + return 0; + sdev->fw_state = SOF_FW_BOOT_PREPARE; /* load the firmware */ @@ -124,15 +165,13 @@ static int sof_resume(struct device *dev, bool runtime_resume) "error: ctx_restore ipc error during resume %d\n", ret); - /* initialize default D0 sub-state */ - sdev->d0_substate = SOF_DSP_D0I0; - return ret; } static int sof_suspend(struct device *dev, bool runtime_suspend) { struct snd_sof_dev *sdev = dev_get_drvdata(dev); + u32 target_state = 0; int ret; /* do nothing if dsp suspend callback is not set */ @@ -140,10 +179,7 @@ static int sof_suspend(struct device *dev, bool runtime_suspend) return 0; if (sdev->fw_state != SOF_FW_BOOT_COMPLETE) - goto power_down; - - /* release trace */ - snd_sof_release_trace(sdev); + goto suspend; /* set restore_stream for all streams during system suspend */ if (!runtime_suspend) { @@ -156,6 +192,15 @@ static int sof_suspend(struct device *dev, bool runtime_suspend) } } + target_state = snd_sof_dsp_power_target(sdev); + + /* Skip to platform-specific suspend if DSP is entering D0 */ + if (target_state == SOF_DSP_PM_D0) + goto suspend; + + /* release trace */ + snd_sof_release_trace(sdev); + #if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE) /* cache debugfs contents during runtime suspend */ if (runtime_suspend) @@ -179,22 +224,26 @@ static int sof_suspend(struct device *dev, bool runtime_suspend) ret); } -power_down: +suspend: /* return if the DSP was not probed successfully */ if (sdev->fw_state == SOF_FW_BOOT_NOT_STARTED) return 0; - /* power down all DSP cores */ + /* platform-specific suspend */ if (runtime_suspend) ret = snd_sof_dsp_runtime_suspend(sdev); else - ret = snd_sof_dsp_suspend(sdev); + ret = snd_sof_dsp_suspend(sdev, target_state); if (ret < 0) dev_err(sdev->dev, "error: failed to power down DSP during suspend %d\n", ret); + /* Do not reset FW state if DSP is in D0 */ + if (target_state == SOF_DSP_PM_D0) + return ret; + /* reset FW state */ sdev->fw_state = SOF_FW_BOOT_NOT_STARTED; @@ -221,112 +270,14 @@ int snd_sof_runtime_resume(struct device *dev) } EXPORT_SYMBOL(snd_sof_runtime_resume); -int snd_sof_set_d0_substate(struct snd_sof_dev *sdev, - enum sof_d0_substate d0_substate) -{ - int ret; - - if (sdev->d0_substate == d0_substate) - return 0; - - /* do platform specific set_state */ - ret = snd_sof_dsp_set_power_state(sdev, d0_substate); - if (ret < 0) - return ret; - - /* update dsp D0 sub-state */ - sdev->d0_substate = d0_substate; - - return 0; -} -EXPORT_SYMBOL(snd_sof_set_d0_substate); - -/* - * Audio DSP states may transform as below:- - * - * D0I3 compatible stream - * Runtime +---------------------+ opened only, timeout - * suspend | +--------------------+ - * +------------+ D0(active) | | - * | | <---------------+ | - * | +--------> | | | - * | |Runtime +--^--+---------^--+--+ The last | | - * | |resume | | | | opened D0I3 | | - * | | | | | | compatible | | - * | | resume| | | | stream closed | | - * | | from | | D3 | | | | - * | | D3 | |suspend | | d0i3 | | - * | | | | | |suspend | | - * | | | | | | | | - * | | | | | | | | - * +-v---+-----------+--v-------+ | | +------+----v----+ - * | | | +-----------> | - * | D3 (suspended) | | | D0I3 +-----+ - * | | +--------------+ | | - * | | resume from | | | - * +-------------------^--------+ d0i3 suspend +----------------+ | - * | | - * | D3 suspend | - * +------------------------------------------------+ - * - * d0i3_suspend = s0_suspend && D0I3 stream opened, - * D3 suspend = !d0i3_suspend, - */ - int snd_sof_resume(struct device *dev) { - struct snd_sof_dev *sdev = dev_get_drvdata(dev); - int ret; - - if (snd_sof_dsp_d0i3_on_suspend(sdev)) { - /* resume from D0I3 */ - dev_dbg(sdev->dev, "DSP will exit from D0i3...\n"); - ret = snd_sof_set_d0_substate(sdev, SOF_DSP_D0I0); - if (ret == -ENOTSUPP) { - /* fallback to resume from D3 */ - dev_dbg(sdev->dev, "D0i3 not supported, fall back to resume from D3...\n"); - goto d3_resume; - } else if (ret < 0) { - dev_err(sdev->dev, "error: failed to exit from D0I3 %d\n", - ret); - return ret; - } - - /* platform-specific resume from D0i3 */ - return snd_sof_dsp_resume(sdev); - } - -d3_resume: - /* resume from D3 */ return sof_resume(dev, false); } EXPORT_SYMBOL(snd_sof_resume); int snd_sof_suspend(struct device *dev) { - struct snd_sof_dev *sdev = dev_get_drvdata(dev); - int ret; - - if (snd_sof_dsp_d0i3_on_suspend(sdev)) { - /* suspend to D0i3 */ - dev_dbg(sdev->dev, "DSP is trying to enter D0i3...\n"); - ret = snd_sof_set_d0_substate(sdev, SOF_DSP_D0I3); - if (ret == -ENOTSUPP) { - /* fallback to D3 suspend */ - dev_dbg(sdev->dev, "D0i3 not supported, fall back to D3...\n"); - goto d3_suspend; - } else if (ret < 0) { - dev_err(sdev->dev, "error: failed to enter D0I3, %d\n", - ret); - return ret; - } - - /* platform-specific suspend to D0i3 */ - return snd_sof_dsp_suspend(sdev); - } - -d3_suspend: - /* suspend to D3 */ return sof_suspend(dev, false); } EXPORT_SYMBOL(snd_sof_suspend); @@ -336,10 +287,13 @@ int snd_sof_prepare(struct device *dev) struct snd_sof_dev *sdev = dev_get_drvdata(dev); #if defined(CONFIG_ACPI) - sdev->s0_suspend = acpi_target_system_state() == ACPI_STATE_S0; + if (acpi_target_system_state() == ACPI_STATE_S0) + sdev->system_suspend_target = SOF_SUSPEND_S0IX; + else + sdev->system_suspend_target = SOF_SUSPEND_S3; #else /* will suspend to S3 by default */ - sdev->s0_suspend = false; + sdev->system_suspend_target = SOF_SUSPEND_S3; #endif return 0; @@ -350,6 +304,6 @@ void snd_sof_complete(struct device *dev) { struct snd_sof_dev *sdev = dev_get_drvdata(dev); - sdev->s0_suspend = false; + sdev->system_suspend_target = SOF_SUSPEND_NONE; } EXPORT_SYMBOL(snd_sof_complete); diff --git a/sound/soc/sof/probe.c b/sound/soc/sof/probe.c new file mode 100644 index 000000000000..c38169fe00c5 --- /dev/null +++ b/sound/soc/sof/probe.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2019-2020 Intel Corporation. All rights reserved. +// +// Author: Cezary Rojewski +// + +#include "sof-priv.h" +#include "probe.h" + +/** + * sof_ipc_probe_init - initialize data probing + * @sdev: SOF sound device + * @stream_tag: Extractor stream tag + * @buffer_size: DMA buffer size to set for extractor + * + * Host chooses whether extraction is supported or not by providing + * valid stream tag to DSP. Once specified, stream described by that + * tag will be tied to DSP for extraction for the entire lifetime of + * probe. + * + * Probing is initialized only once and each INIT request must be + * matched by DEINIT call. + */ +int sof_ipc_probe_init(struct snd_sof_dev *sdev, + u32 stream_tag, size_t buffer_size) +{ + struct sof_ipc_probe_dma_add_params *msg; + struct sof_ipc_reply reply; + size_t size = struct_size(msg, dma, 1); + int ret; + + msg = kmalloc(size, GFP_KERNEL); + if (!msg) + return -ENOMEM; + msg->hdr.size = size; + msg->hdr.cmd = SOF_IPC_GLB_PROBE | SOF_IPC_PROBE_INIT; + msg->num_elems = 1; + msg->dma[0].stream_tag = stream_tag; + msg->dma[0].dma_buffer_size = buffer_size; + + ret = sof_ipc_tx_message(sdev->ipc, msg->hdr.cmd, msg, msg->hdr.size, + &reply, sizeof(reply)); + kfree(msg); + return ret; +} +EXPORT_SYMBOL(sof_ipc_probe_init); + +/** + * sof_ipc_probe_deinit - cleanup after data probing + * @sdev: SOF sound device + * + * Host sends DEINIT request to free previously initialized probe + * on DSP side once it is no longer needed. DEINIT only when there + * are no probes connected and with all injectors detached. + */ +int sof_ipc_probe_deinit(struct snd_sof_dev *sdev) +{ + struct sof_ipc_cmd_hdr msg; + struct sof_ipc_reply reply; + + msg.size = sizeof(msg); + msg.cmd = SOF_IPC_GLB_PROBE | SOF_IPC_PROBE_DEINIT; + + return sof_ipc_tx_message(sdev->ipc, msg.cmd, &msg, msg.size, + &reply, sizeof(reply)); +} +EXPORT_SYMBOL(sof_ipc_probe_deinit); + +static int sof_ipc_probe_info(struct snd_sof_dev *sdev, unsigned int cmd, + void **params, size_t *num_params) +{ + struct sof_ipc_probe_info_params msg = {{{0}}}; + struct sof_ipc_probe_info_params *reply; + size_t bytes; + int ret; + + *params = NULL; + *num_params = 0; + + reply = kzalloc(SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL); + if (!reply) + return -ENOMEM; + msg.rhdr.hdr.size = sizeof(msg); + msg.rhdr.hdr.cmd = SOF_IPC_GLB_PROBE | cmd; + + ret = sof_ipc_tx_message(sdev->ipc, msg.rhdr.hdr.cmd, &msg, + msg.rhdr.hdr.size, reply, SOF_IPC_MSG_MAX_SIZE); + if (ret < 0 || reply->rhdr.error < 0) + goto exit; + + if (!reply->num_elems) + goto exit; + + if (cmd == SOF_IPC_PROBE_DMA_INFO) + bytes = sizeof(reply->dma[0]); + else + bytes = sizeof(reply->desc[0]); + bytes *= reply->num_elems; + *params = kmemdup(&reply->dma[0], bytes, GFP_KERNEL); + if (!*params) { + ret = -ENOMEM; + goto exit; + } + *num_params = reply->num_elems; + +exit: + kfree(reply); + return ret; +} + +/** + * sof_ipc_probe_dma_info - retrieve list of active injection dmas + * @sdev: SOF sound device + * @dma: Returned list of active dmas + * @num_dma: Returned count of active dmas + * + * Host sends DMA_INFO request to obtain list of injection dmas it + * can use to transfer data over with. + * + * Note that list contains only injection dmas as there is only one + * extractor (dma) and it is always assigned on probing init. + * DSP knows exactly where data from extraction probes is going to, + * which is not the case for injection where multiple streams + * could be engaged. + */ +int sof_ipc_probe_dma_info(struct snd_sof_dev *sdev, + struct sof_probe_dma **dma, size_t *num_dma) +{ + return sof_ipc_probe_info(sdev, SOF_IPC_PROBE_DMA_INFO, + (void **)dma, num_dma); +} +EXPORT_SYMBOL(sof_ipc_probe_dma_info); + +/** + * sof_ipc_probe_dma_add - attach to specified dmas + * @sdev: SOF sound device + * @dma: List of streams (dmas) to attach to + * @num_dma: Number of elements in @dma + * + * Contrary to extraction, injection streams are never assigned + * on init. Before attempting any data injection, host is responsible + * for specifying streams which will be later used to transfer data + * to connected probe points. + */ +int sof_ipc_probe_dma_add(struct snd_sof_dev *sdev, + struct sof_probe_dma *dma, size_t num_dma) +{ + struct sof_ipc_probe_dma_add_params *msg; + struct sof_ipc_reply reply; + size_t size = struct_size(msg, dma, num_dma); + int ret; + + msg = kmalloc(size, GFP_KERNEL); + if (!msg) + return -ENOMEM; + msg->hdr.size = size; + msg->num_elems = num_dma; + msg->hdr.cmd = SOF_IPC_GLB_PROBE | SOF_IPC_PROBE_DMA_ADD; + memcpy(&msg->dma[0], dma, size - sizeof(*msg)); + + ret = sof_ipc_tx_message(sdev->ipc, msg->hdr.cmd, msg, msg->hdr.size, + &reply, sizeof(reply)); + kfree(msg); + return ret; +} +EXPORT_SYMBOL(sof_ipc_probe_dma_add); + +/** + * sof_ipc_probe_dma_remove - detach from specified dmas + * @sdev: SOF sound device + * @stream_tag: List of stream tags to detach from + * @num_stream_tag: Number of elements in @stream_tag + * + * Host sends DMA_REMOVE request to free previously attached stream + * from being occupied for injection. Each detach operation should + * match equivalent DMA_ADD. Detach only when all probes tied to + * given stream have been disconnected. + */ +int sof_ipc_probe_dma_remove(struct snd_sof_dev *sdev, + unsigned int *stream_tag, size_t num_stream_tag) +{ + struct sof_ipc_probe_dma_remove_params *msg; + struct sof_ipc_reply reply; + size_t size = struct_size(msg, stream_tag, num_stream_tag); + int ret; + + msg = kmalloc(size, GFP_KERNEL); + if (!msg) + return -ENOMEM; + msg->hdr.size = size; + msg->num_elems = num_stream_tag; + msg->hdr.cmd = SOF_IPC_GLB_PROBE | SOF_IPC_PROBE_DMA_REMOVE; + memcpy(&msg->stream_tag[0], stream_tag, size - sizeof(*msg)); + + ret = sof_ipc_tx_message(sdev->ipc, msg->hdr.cmd, msg, msg->hdr.size, + &reply, sizeof(reply)); + kfree(msg); + return ret; +} +EXPORT_SYMBOL(sof_ipc_probe_dma_remove); + +/** + * sof_ipc_probe_points_info - retrieve list of active probe points + * @sdev: SOF sound device + * @desc: Returned list of active probes + * @num_desc: Returned count of active probes + * + * Host sends PROBE_POINT_INFO request to obtain list of active probe + * points, valid for disconnection when given probe is no longer + * required. + */ +int sof_ipc_probe_points_info(struct snd_sof_dev *sdev, + struct sof_probe_point_desc **desc, size_t *num_desc) +{ + return sof_ipc_probe_info(sdev, SOF_IPC_PROBE_POINT_INFO, + (void **)desc, num_desc); +} +EXPORT_SYMBOL(sof_ipc_probe_points_info); + +/** + * sof_ipc_probe_points_add - connect specified probes + * @sdev: SOF sound device + * @desc: List of probe points to connect + * @num_desc: Number of elements in @desc + * + * Dynamically connects to provided set of endpoints. Immediately + * after connection is established, host must be prepared to + * transfer data from or to target stream given the probing purpose. + * + * Each probe point should be removed using PROBE_POINT_REMOVE + * request when no longer needed. + */ +int sof_ipc_probe_points_add(struct snd_sof_dev *sdev, + struct sof_probe_point_desc *desc, size_t num_desc) +{ + struct sof_ipc_probe_point_add_params *msg; + struct sof_ipc_reply reply; + size_t size = struct_size(msg, desc, num_desc); + int ret; + + msg = kmalloc(size, GFP_KERNEL); + if (!msg) + return -ENOMEM; + msg->hdr.size = size; + msg->num_elems = num_desc; + msg->hdr.cmd = SOF_IPC_GLB_PROBE | SOF_IPC_PROBE_POINT_ADD; + memcpy(&msg->desc[0], desc, size - sizeof(*msg)); + + ret = sof_ipc_tx_message(sdev->ipc, msg->hdr.cmd, msg, msg->hdr.size, + &reply, sizeof(reply)); + kfree(msg); + return ret; +} +EXPORT_SYMBOL(sof_ipc_probe_points_add); + +/** + * sof_ipc_probe_points_remove - disconnect specified probes + * @sdev: SOF sound device + * @buffer_id: List of probe points to disconnect + * @num_buffer_id: Number of elements in @desc + * + * Removes previously connected probes from list of active probe + * points and frees all resources on DSP side. + */ +int sof_ipc_probe_points_remove(struct snd_sof_dev *sdev, + unsigned int *buffer_id, size_t num_buffer_id) +{ + struct sof_ipc_probe_point_remove_params *msg; + struct sof_ipc_reply reply; + size_t size = struct_size(msg, buffer_id, num_buffer_id); + int ret; + + msg = kmalloc(size, GFP_KERNEL); + if (!msg) + return -ENOMEM; + msg->hdr.size = size; + msg->num_elems = num_buffer_id; + msg->hdr.cmd = SOF_IPC_GLB_PROBE | SOF_IPC_PROBE_POINT_REMOVE; + memcpy(&msg->buffer_id[0], buffer_id, size - sizeof(*msg)); + + ret = sof_ipc_tx_message(sdev->ipc, msg->hdr.cmd, msg, msg->hdr.size, + &reply, sizeof(reply)); + kfree(msg); + return ret; +} +EXPORT_SYMBOL(sof_ipc_probe_points_remove); diff --git a/sound/soc/sof/probe.h b/sound/soc/sof/probe.h new file mode 100644 index 000000000000..45daa5552834 --- /dev/null +++ b/sound/soc/sof/probe.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * Copyright(c) 2019-2020 Intel Corporation. All rights reserved. + * + * Author: Cezary Rojewski + */ + +#ifndef __SOF_PROBE_H +#define __SOF_PROBE_H + +#include + +struct snd_sof_dev; + +#define SOF_PROBE_INVALID_NODE_ID UINT_MAX + +struct sof_probe_dma { + unsigned int stream_tag; + unsigned int dma_buffer_size; +} __packed; + +enum sof_connection_purpose { + SOF_CONNECTION_PURPOSE_EXTRACT = 1, + SOF_CONNECTION_PURPOSE_INJECT, +}; + +struct sof_probe_point_desc { + unsigned int buffer_id; + unsigned int purpose; + unsigned int stream_tag; +} __packed; + +struct sof_ipc_probe_dma_add_params { + struct sof_ipc_cmd_hdr hdr; + unsigned int num_elems; + struct sof_probe_dma dma[0]; +} __packed; + +struct sof_ipc_probe_info_params { + struct sof_ipc_reply rhdr; + unsigned int num_elems; + union { + struct sof_probe_dma dma[0]; + struct sof_probe_point_desc desc[0]; + }; +} __packed; + +struct sof_ipc_probe_dma_remove_params { + struct sof_ipc_cmd_hdr hdr; + unsigned int num_elems; + unsigned int stream_tag[0]; +} __packed; + +struct sof_ipc_probe_point_add_params { + struct sof_ipc_cmd_hdr hdr; + unsigned int num_elems; + struct sof_probe_point_desc desc[0]; +} __packed; + +struct sof_ipc_probe_point_remove_params { + struct sof_ipc_cmd_hdr hdr; + unsigned int num_elems; + unsigned int buffer_id[0]; +} __packed; + +int sof_ipc_probe_init(struct snd_sof_dev *sdev, + u32 stream_tag, size_t buffer_size); +int sof_ipc_probe_deinit(struct snd_sof_dev *sdev); +int sof_ipc_probe_dma_info(struct snd_sof_dev *sdev, + struct sof_probe_dma **dma, size_t *num_dma); +int sof_ipc_probe_dma_add(struct snd_sof_dev *sdev, + struct sof_probe_dma *dma, size_t num_dma); +int sof_ipc_probe_dma_remove(struct snd_sof_dev *sdev, + unsigned int *stream_tag, size_t num_stream_tag); +int sof_ipc_probe_points_info(struct snd_sof_dev *sdev, + struct sof_probe_point_desc **desc, size_t *num_desc); +int sof_ipc_probe_points_add(struct snd_sof_dev *sdev, + struct sof_probe_point_desc *desc, size_t num_desc); +int sof_ipc_probe_points_remove(struct snd_sof_dev *sdev, + unsigned int *buffer_id, size_t num_buffer_id); + +#endif diff --git a/sound/soc/sof/sof-audio.c b/sound/soc/sof/sof-audio.c index 0d8f65b9ae25..fc4ed2a8a914 100644 --- a/sound/soc/sof/sof-audio.c +++ b/sound/soc/sof/sof-audio.c @@ -11,7 +11,40 @@ #include "sof-audio.h" #include "ops.h" -bool snd_sof_dsp_d0i3_on_suspend(struct snd_sof_dev *sdev) +/* + * helper to determine if there are only D0i3 compatible + * streams active + */ +bool snd_sof_dsp_only_d0i3_compatible_stream_active(struct snd_sof_dev *sdev) +{ + struct snd_pcm_substream *substream; + struct snd_sof_pcm *spcm; + bool d0i3_compatible_active = false; + int dir; + + list_for_each_entry(spcm, &sdev->pcm_list, list) { + for_each_pcm_streams(dir) { + substream = spcm->stream[dir].substream; + if (!substream || !substream->runtime) + continue; + + /* + * substream->runtime being not NULL indicates that + * that the stream is open. No need to check the + * stream state. + */ + if (!spcm->stream[dir].d0i3_compatible) + return false; + + d0i3_compatible_active = true; + } + } + + return d0i3_compatible_active; +} +EXPORT_SYMBOL(snd_sof_dsp_only_d0i3_compatible_stream_active); + +bool snd_sof_stream_suspend_ignored(struct snd_sof_dev *sdev) { struct snd_sof_pcm *spcm; @@ -38,7 +71,14 @@ int sof_set_hw_params_upon_resume(struct device *dev) * have been suspended. */ list_for_each_entry(spcm, &sdev->pcm_list, list) { - for (dir = 0; dir <= SNDRV_PCM_STREAM_CAPTURE; dir++) { + for_each_pcm_streams(dir) { + /* + * do not reset hw_params upon resume for streams that + * were kept running during suspend + */ + if (spcm->stream[dir].suspend_ignored) + continue; + substream = spcm->stream[dir].substream; if (!substream || !substream->runtime) continue; @@ -279,16 +319,11 @@ struct snd_sof_pcm *snd_sof_find_spcm_comp(struct snd_soc_component *scomp, int dir; list_for_each_entry(spcm, &sdev->pcm_list, list) { - dir = SNDRV_PCM_STREAM_PLAYBACK; - if (spcm->stream[dir].comp_id == comp_id) { - *direction = dir; - return spcm; - } - - dir = SNDRV_PCM_STREAM_CAPTURE; - if (spcm->stream[dir].comp_id == comp_id) { - *direction = dir; - return spcm; + for_each_pcm_streams(dir) { + if (spcm->stream[dir].comp_id == comp_id) { + *direction = dir; + return spcm; + } } } diff --git a/sound/soc/sof/sof-audio.h b/sound/soc/sof/sof-audio.h index a62fb2da6a6e..bf65f31af858 100644 --- a/sound/soc/sof/sof-audio.h +++ b/sound/soc/sof/sof-audio.h @@ -11,6 +11,8 @@ #ifndef __SOUND_SOC_SOF_AUDIO_H #define __SOUND_SOC_SOF_AUDIO_H +#include + #include #include #include /* needs to be included before control.h */ @@ -189,6 +191,7 @@ struct snd_sof_pcm *snd_sof_find_spcm_comp(struct snd_soc_component *scomp, struct snd_sof_pcm *snd_sof_find_spcm_pcm_id(struct snd_soc_component *scomp, unsigned int pcm_id); void snd_sof_pcm_period_elapsed(struct snd_pcm_substream *substream); +void snd_sof_pcm_period_elapsed_work(struct work_struct *work); /* * Mixer IPC @@ -202,7 +205,8 @@ int snd_sof_ipc_set_get_comp_data(struct snd_sof_control *scontrol, /* PM */ int sof_restore_pipelines(struct device *dev); int sof_set_hw_params_upon_resume(struct device *dev); -bool snd_sof_dsp_d0i3_on_suspend(struct snd_sof_dev *sdev); +bool snd_sof_stream_suspend_ignored(struct snd_sof_dev *sdev); +bool snd_sof_dsp_only_d0i3_compatible_stream_active(struct snd_sof_dev *sdev); /* Machine driver enumeration */ int sof_machine_register(struct snd_sof_dev *sdev, void *pdata); diff --git a/sound/soc/sof/sof-of-dev.c b/sound/soc/sof/sof-of-dev.c index 39ea8af6213f..16e49f2ee629 100644 --- a/sound/soc/sof/sof-of-dev.c +++ b/sound/soc/sof/sof-of-dev.c @@ -13,10 +13,19 @@ #include "ops.h" extern struct snd_sof_dsp_ops sof_imx8_ops; +extern struct snd_sof_dsp_ops sof_imx8x_ops; /* platform specific devices */ #if IS_ENABLED(CONFIG_SND_SOC_SOF_IMX8) static struct sof_dev_desc sof_of_imx8qxp_desc = { + .default_fw_path = "imx/sof", + .default_tplg_path = "imx/sof-tplg", + .default_fw_filename = "sof-imx8x.ri", + .nocodec_tplg_filename = "sof-imx8-nocodec.tplg", + .ops = &sof_imx8x_ops, +}; + +static struct sof_dev_desc sof_of_imx8qm_desc = { .default_fw_path = "imx/sof", .default_tplg_path = "imx/sof-tplg", .default_fw_filename = "sof-imx8.ri", @@ -103,6 +112,7 @@ static int sof_of_remove(struct platform_device *pdev) static const struct of_device_id sof_of_ids[] = { #if IS_ENABLED(CONFIG_SND_SOC_SOF_IMX8) { .compatible = "fsl,imx8qxp-dsp", .data = &sof_of_imx8qxp_desc}, + { .compatible = "fsl,imx8qm-dsp", .data = &sof_of_imx8qm_desc}, #endif { } }; diff --git a/sound/soc/sof/sof-priv.h b/sound/soc/sof/sof-priv.h index bc2337cf1142..a4b297c842df 100644 --- a/sound/soc/sof/sof-priv.h +++ b/sound/soc/sof/sof-priv.h @@ -54,10 +54,26 @@ extern int sof_core_debug; (IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE) || \ IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_IPC_FLOOD_TEST)) -/* DSP D0ix sub-state */ -enum sof_d0_substate { - SOF_DSP_D0I0 = 0, /* DSP default D0 substate */ - SOF_DSP_D0I3, /* DSP D0i3(low power) substate*/ +/* DSP power state */ +enum sof_dsp_power_states { + SOF_DSP_PM_D0, + SOF_DSP_PM_D1, + SOF_DSP_PM_D2, + SOF_DSP_PM_D3_HOT, + SOF_DSP_PM_D3, + SOF_DSP_PM_D3_COLD, +}; + +struct sof_dsp_power_state { + u32 state; + u32 substate; /* platform-specific */ +}; + +/* System suspend target state */ +enum sof_system_suspend_state { + SOF_SUSPEND_NONE = 0, + SOF_SUSPEND_S0IX, + SOF_SUSPEND_S3, }; struct snd_sof_dev; @@ -154,6 +170,27 @@ struct snd_sof_dsp_ops { snd_pcm_uframes_t (*pcm_pointer)(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream); /* optional */ +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_PROBES) + /* Except for probe_pointer, all probe ops are mandatory */ + int (*probe_assign)(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_soc_dai *dai); /* mandatory */ + int (*probe_free)(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_soc_dai *dai); /* mandatory */ + int (*probe_set_params)(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_compr_params *params, + struct snd_soc_dai *dai); /* mandatory */ + int (*probe_trigger)(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, int cmd, + struct snd_soc_dai *dai); /* mandatory */ + int (*probe_pointer)(struct snd_sof_dev *sdev, + struct snd_compr_stream *cstream, + struct snd_compr_tstamp *tstamp, + struct snd_soc_dai *dai); /* optional */ +#endif + /* host read DSP stream data */ void (*ipc_msg_data)(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream, @@ -169,14 +206,15 @@ struct snd_sof_dsp_ops { int (*post_fw_run)(struct snd_sof_dev *sof_dev); /* optional */ /* DSP PM */ - int (*suspend)(struct snd_sof_dev *sof_dev); /* optional */ + int (*suspend)(struct snd_sof_dev *sof_dev, + u32 target_state); /* optional */ int (*resume)(struct snd_sof_dev *sof_dev); /* optional */ int (*runtime_suspend)(struct snd_sof_dev *sof_dev); /* optional */ int (*runtime_resume)(struct snd_sof_dev *sof_dev); /* optional */ int (*runtime_idle)(struct snd_sof_dev *sof_dev); /* optional */ int (*set_hw_params_upon_resume)(struct snd_sof_dev *sdev); /* optional */ int (*set_power_state)(struct snd_sof_dev *sdev, - enum sof_d0_substate d0_substate); /* optional */ + const struct sof_dsp_power_state *target_state); /* optional */ /* DSP clocking */ int (*set_clk)(struct snd_sof_dev *sof_dev, u32 freq); /* optional */ @@ -323,10 +361,11 @@ struct snd_sof_dev { */ struct snd_soc_component_driver plat_drv; - /* power states related */ - enum sof_d0_substate d0_substate; - /* flag to track if the intended power target of suspend is S0ix */ - bool s0_suspend; + /* current DSP power state */ + struct sof_dsp_power_state dsp_power_state; + + /* Intended power target of system suspend */ + enum sof_system_suspend_state system_suspend_target; /* DSP firmware boot */ wait_queue_head_t boot_wait; @@ -376,16 +415,15 @@ struct snd_sof_dev { u32 enabled_cores_mask; /* keep track of enabled cores */ /* FW configuration */ - struct sof_ipc_dma_buffer_data *info_buffer; struct sof_ipc_window *info_window; /* IPC timeouts in ms */ int ipc_timeout; int boot_timeout; - /* Wait queue for code loading */ - wait_queue_head_t waitq; - int code_loading; +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_PROBES) + unsigned int extractor_stream_tag; +#endif /* DMA for Trace */ struct snd_dma_buffer dmatb; @@ -417,8 +455,6 @@ int snd_sof_resume(struct device *dev); int snd_sof_suspend(struct device *dev); int snd_sof_prepare(struct device *dev); void snd_sof_complete(struct device *dev); -int snd_sof_set_d0_substate(struct snd_sof_dev *sdev, - enum sof_d0_substate d0_substate); void snd_sof_new_platform_drv(struct snd_sof_dev *sdev); @@ -454,6 +490,9 @@ int snd_sof_ipc_valid(struct snd_sof_dev *sdev); int sof_ipc_tx_message(struct snd_sof_ipc *ipc, u32 header, void *msg_data, size_t msg_bytes, void *reply_data, size_t reply_bytes); +int sof_ipc_tx_message_no_pm(struct snd_sof_ipc *ipc, u32 header, + void *msg_data, size_t msg_bytes, + void *reply_data, size_t reply_bytes); /* * Trace/debug diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 9f4f8868b386..fe8ba3e05e08 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -9,6 +9,7 @@ // #include +#include #include #include #include @@ -1240,6 +1241,8 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp, { struct snd_soc_card *card = scomp->card; struct snd_soc_pcm_runtime *rtd; + struct snd_soc_dai *cpu_dai; + int i; list_for_each_entry(rtd, &card->rtd_list, list) { dev_vdbg(scomp->dev, "tplg: check widget: %s stream: %s dai stream: %s\n", @@ -1254,13 +1257,15 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp, switch (w->id) { case snd_soc_dapm_dai_out: - rtd->cpu_dai->capture_widget = w; + for_each_rtd_cpu_dais(rtd, i, cpu_dai) + cpu_dai->capture_widget = w; dai->name = rtd->dai_link->name; dev_dbg(scomp->dev, "tplg: connected widget %s -> DAI link %s\n", w->name, rtd->dai_link->name); break; case snd_soc_dapm_dai_in: - rtd->cpu_dai->playback_widget = w; + for_each_rtd_cpu_dais(rtd, i, cpu_dai) + cpu_dai->playback_widget = w; dai->name = rtd->dai_link->name; dev_dbg(scomp->dev, "tplg: connected widget %s -> DAI link %s\n", w->name, rtd->dai_link->name); @@ -2444,7 +2449,7 @@ static int sof_dai_load(struct snd_soc_component *scomp, int index, struct snd_soc_tplg_stream_caps *caps; struct snd_soc_tplg_private *private = &pcm->priv; struct snd_sof_pcm *spcm; - int stream = SNDRV_PCM_STREAM_PLAYBACK; + int stream; int ret = 0; /* nothing to do for BEs atm */ @@ -2456,8 +2461,12 @@ static int sof_dai_load(struct snd_soc_component *scomp, int index, return -ENOMEM; spcm->scomp = scomp; - spcm->stream[SNDRV_PCM_STREAM_PLAYBACK].comp_id = COMP_ID_UNASSIGNED; - spcm->stream[SNDRV_PCM_STREAM_CAPTURE].comp_id = COMP_ID_UNASSIGNED; + + for_each_pcm_streams(stream) { + spcm->stream[stream].comp_id = COMP_ID_UNASSIGNED; + INIT_WORK(&spcm->stream[stream].period_elapsed_work, + snd_sof_pcm_period_elapsed_work); + } spcm->pcm = *pcm; dev_dbg(scomp->dev, "tplg: load pcm %s\n", pcm->dai_name); @@ -2478,8 +2487,10 @@ static int sof_dai_load(struct snd_soc_component *scomp, int index, if (!spcm->pcm.playback) goto capture; + stream = SNDRV_PCM_STREAM_PLAYBACK; + dev_vdbg(scomp->dev, "tplg: pcm %s stream tokens: playback d0i3:%d\n", - spcm->pcm.pcm_name, spcm->stream[0].d0i3_compatible); + spcm->pcm.pcm_name, spcm->stream[stream].d0i3_compatible); caps = &spcm->pcm.caps[stream]; @@ -2509,7 +2520,7 @@ static int sof_dai_load(struct snd_soc_component *scomp, int index, return ret; dev_vdbg(scomp->dev, "tplg: pcm %s stream tokens: capture d0i3:%d\n", - spcm->pcm.pcm_name, spcm->stream[1].d0i3_compatible); + spcm->pcm.pcm_name, spcm->stream[stream].d0i3_compatible); caps = &spcm->pcm.caps[stream]; diff --git a/sound/soc/sprd/Kconfig b/sound/soc/sprd/Kconfig index 5474fd3de8c0..5e0ac8278572 100644 --- a/sound/soc/sprd/Kconfig +++ b/sound/soc/sprd/Kconfig @@ -8,7 +8,7 @@ config SND_SOC_SPRD the Spreadtrum SoCs' Audio interfaces. config SND_SOC_SPRD_MCDT - bool "Spreadtrum multi-channel data transfer support" + tristate "Spreadtrum multi-channel data transfer support" depends on SND_SOC_SPRD help Say y here to enable multi-channel data transfer support. It diff --git a/sound/soc/sprd/sprd-mcdt.h b/sound/soc/sprd/sprd-mcdt.h index 9cc7e207ac76..679e3af3baad 100644 --- a/sound/soc/sprd/sprd-mcdt.h +++ b/sound/soc/sprd/sprd-mcdt.h @@ -48,7 +48,7 @@ struct sprd_mcdt_chan { struct list_head list; }; -#ifdef CONFIG_SND_SOC_SPRD_MCDT +#if IS_ENABLED(CONFIG_SND_SOC_SPRD_MCDT) struct sprd_mcdt_chan *sprd_mcdt_request_chan(u8 channel, enum sprd_mcdt_channel_type type); void sprd_mcdt_free_chan(struct sprd_mcdt_chan *chan); diff --git a/sound/soc/sprd/sprd-pcm-compress.c b/sound/soc/sprd/sprd-pcm-compress.c index 6cddf551bc11..74d48340cade 100644 --- a/sound/soc/sprd/sprd-pcm-compress.c +++ b/sound/soc/sprd/sprd-pcm-compress.c @@ -135,7 +135,7 @@ static int sprd_platform_compr_dma_config(struct snd_compr_stream *cstream, struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, DRV_NAME); struct device *dev = component->dev; - struct sprd_compr_data *data = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct sprd_compr_data *data = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct sprd_pcm_dma_params *dma_params = data->dma_params; struct sprd_compr_dma *dma = &stream->dma[channel]; struct dma_slave_config config = { }; @@ -321,7 +321,7 @@ static int sprd_platform_compr_open(struct snd_compr_stream *cstream) struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, DRV_NAME); struct device *dev = component->dev; - struct sprd_compr_data *data = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct sprd_compr_data *data = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct sprd_compr_stream *stream; struct sprd_compr_callback cb; int stream_id = cstream->direction, ret; diff --git a/sound/soc/sprd/sprd-pcm-dma.c b/sound/soc/sprd/sprd-pcm-dma.c index 2284558684bc..d12d3cad8cbd 100644 --- a/sound/soc/sprd/sprd-pcm-dma.c +++ b/sound/soc/sprd/sprd-pcm-dma.c @@ -200,7 +200,7 @@ static int sprd_pcm_hw_params(struct snd_soc_component *component, unsigned long flags; int ret, i, j, sg_num; - dma_params = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + dma_params = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); if (!dma_params) { dev_warn(component->dev, "no dma parameters setting\n"); dma_private->params = NULL; diff --git a/sound/soc/stm/stm32_adfsdm.c b/sound/soc/stm/stm32_adfsdm.c index 51407a21c440..16ff02953015 100644 --- a/sound/soc/stm/stm32_adfsdm.c +++ b/sound/soc/stm/stm32_adfsdm.c @@ -215,7 +215,7 @@ static int stm32_adfsdm_trigger(struct snd_soc_component *component, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct stm32_adfsdm_priv *priv = - snd_soc_dai_get_drvdata(rtd->cpu_dai); + snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); switch (cmd) { case SNDRV_PCM_TRIGGER_START: @@ -235,7 +235,7 @@ static int stm32_adfsdm_pcm_open(struct snd_soc_component *component, struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct stm32_adfsdm_priv *priv = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct stm32_adfsdm_priv *priv = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); int ret; ret = snd_soc_set_runtime_hwparams(substream, &stm32_adfsdm_pcm_hw); @@ -250,7 +250,7 @@ static int stm32_adfsdm_pcm_close(struct snd_soc_component *component, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct stm32_adfsdm_priv *priv = - snd_soc_dai_get_drvdata(rtd->cpu_dai); + snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); priv->substream = NULL; @@ -263,7 +263,7 @@ static snd_pcm_uframes_t stm32_adfsdm_pcm_pointer( { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct stm32_adfsdm_priv *priv = - snd_soc_dai_get_drvdata(rtd->cpu_dai); + snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); return bytes_to_frames(substream->runtime, priv->pos); } @@ -274,7 +274,7 @@ static int stm32_adfsdm_pcm_hw_params(struct snd_soc_component *component, { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct stm32_adfsdm_priv *priv = - snd_soc_dai_get_drvdata(rtd->cpu_dai); + snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); priv->pcm_buff = substream->runtime->dma_area; @@ -287,7 +287,7 @@ static int stm32_adfsdm_pcm_new(struct snd_soc_component *component, { struct snd_pcm *pcm = rtd->pcm; struct stm32_adfsdm_priv *priv = - snd_soc_dai_get_drvdata(rtd->cpu_dai); + snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); unsigned int size = DFSDM_MAX_PERIODS * DFSDM_MAX_PERIOD_SIZE; snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV, diff --git a/sound/soc/stm/stm32_i2s.c b/sound/soc/stm/stm32_i2s.c index 3e7226a53e53..7c4d63c33f15 100644 --- a/sound/soc/stm/stm32_i2s.c +++ b/sound/soc/stm/stm32_i2s.c @@ -831,25 +831,33 @@ static int stm32_i2s_parse_dt(struct platform_device *pdev, /* Get clocks */ i2s->pclk = devm_clk_get(&pdev->dev, "pclk"); if (IS_ERR(i2s->pclk)) { - dev_err(&pdev->dev, "Could not get pclk\n"); + if (PTR_ERR(i2s->pclk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Could not get pclk: %ld\n", + PTR_ERR(i2s->pclk)); return PTR_ERR(i2s->pclk); } i2s->i2sclk = devm_clk_get(&pdev->dev, "i2sclk"); if (IS_ERR(i2s->i2sclk)) { - dev_err(&pdev->dev, "Could not get i2sclk\n"); + if (PTR_ERR(i2s->i2sclk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Could not get i2sclk: %ld\n", + PTR_ERR(i2s->i2sclk)); return PTR_ERR(i2s->i2sclk); } i2s->x8kclk = devm_clk_get(&pdev->dev, "x8k"); if (IS_ERR(i2s->x8kclk)) { - dev_err(&pdev->dev, "missing x8k parent clock\n"); + if (PTR_ERR(i2s->x8kclk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Could not get x8k parent clock: %ld\n", + PTR_ERR(i2s->x8kclk)); return PTR_ERR(i2s->x8kclk); } i2s->x11kclk = devm_clk_get(&pdev->dev, "x11k"); if (IS_ERR(i2s->x11kclk)) { - dev_err(&pdev->dev, "missing x11k parent clock\n"); + if (PTR_ERR(i2s->x11kclk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Could not get x11k parent clock: %ld\n", + PTR_ERR(i2s->x11kclk)); return PTR_ERR(i2s->x11kclk); } @@ -866,12 +874,24 @@ static int stm32_i2s_parse_dt(struct platform_device *pdev, } /* Reset */ - rst = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (!IS_ERR(rst)) { - reset_control_assert(rst); - udelay(2); - reset_control_deassert(rst); + rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); + if (IS_ERR(rst)) { + if (PTR_ERR(rst) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Reset controller error %ld\n", + PTR_ERR(rst)); + return PTR_ERR(rst); } + reset_control_assert(rst); + udelay(2); + reset_control_deassert(rst); + + return 0; +} + +static int stm32_i2s_remove(struct platform_device *pdev) +{ + snd_dmaengine_pcm_unregister(&pdev->dev); + snd_soc_unregister_component(&pdev->dev); return 0; } @@ -903,48 +923,62 @@ static int stm32_i2s_probe(struct platform_device *pdev) i2s->regmap = devm_regmap_init_mmio_clk(&pdev->dev, "pclk", i2s->base, i2s->regmap_conf); if (IS_ERR(i2s->regmap)) { - dev_err(&pdev->dev, "regmap init failed\n"); + if (PTR_ERR(i2s->regmap) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Regmap init error %ld\n", + PTR_ERR(i2s->regmap)); return PTR_ERR(i2s->regmap); } - ret = devm_snd_soc_register_component(&pdev->dev, &stm32_i2s_component, - i2s->dai_drv, 1); - if (ret) + ret = snd_dmaengine_pcm_register(&pdev->dev, &stm32_i2s_pcm_config, 0); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "PCM DMA register error %d\n", ret); return ret; + } - ret = devm_snd_dmaengine_pcm_register(&pdev->dev, - &stm32_i2s_pcm_config, 0); - if (ret) + ret = snd_soc_register_component(&pdev->dev, &stm32_i2s_component, + i2s->dai_drv, 1); + if (ret) { + snd_dmaengine_pcm_unregister(&pdev->dev); return ret; + } /* Set SPI/I2S in i2s mode */ ret = regmap_update_bits(i2s->regmap, STM32_I2S_CGFR_REG, I2S_CGFR_I2SMOD, I2S_CGFR_I2SMOD); if (ret) - return ret; + goto error; ret = regmap_read(i2s->regmap, STM32_I2S_IPIDR_REG, &val); if (ret) - return ret; + goto error; if (val == I2S_IPIDR_NUMBER) { ret = regmap_read(i2s->regmap, STM32_I2S_HWCFGR_REG, &val); if (ret) - return ret; + goto error; if (!FIELD_GET(I2S_HWCFGR_I2S_SUPPORT_MASK, val)) { dev_err(&pdev->dev, "Device does not support i2s mode\n"); - return -EPERM; + ret = -EPERM; + goto error; } ret = regmap_read(i2s->regmap, STM32_I2S_VERR_REG, &val); + if (ret) + goto error; dev_dbg(&pdev->dev, "I2S version: %lu.%lu registered\n", FIELD_GET(I2S_VERR_MAJ_MASK, val), FIELD_GET(I2S_VERR_MIN_MASK, val)); } + return ret; + +error: + stm32_i2s_remove(pdev); + return ret; } @@ -981,6 +1015,7 @@ static struct platform_driver stm32_i2s_driver = { .pm = &stm32_i2s_pm_ops, }, .probe = stm32_i2s_probe, + .remove = stm32_i2s_remove, }; module_platform_driver(stm32_i2s_driver); diff --git a/sound/soc/stm/stm32_sai.c b/sound/soc/stm/stm32_sai.c index e20267504b16..058757c721f0 100644 --- a/sound/soc/stm/stm32_sai.c +++ b/sound/soc/stm/stm32_sai.c @@ -174,20 +174,26 @@ static int stm32_sai_probe(struct platform_device *pdev) if (!STM_SAI_IS_F4(sai)) { sai->pclk = devm_clk_get(&pdev->dev, "pclk"); if (IS_ERR(sai->pclk)) { - dev_err(&pdev->dev, "missing bus clock pclk\n"); + if (PTR_ERR(sai->pclk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "missing bus clock pclk: %ld\n", + PTR_ERR(sai->pclk)); return PTR_ERR(sai->pclk); } } sai->clk_x8k = devm_clk_get(&pdev->dev, "x8k"); if (IS_ERR(sai->clk_x8k)) { - dev_err(&pdev->dev, "missing x8k parent clock\n"); + if (PTR_ERR(sai->clk_x8k) != -EPROBE_DEFER) + dev_err(&pdev->dev, "missing x8k parent clock: %ld\n", + PTR_ERR(sai->clk_x8k)); return PTR_ERR(sai->clk_x8k); } sai->clk_x11k = devm_clk_get(&pdev->dev, "x11k"); if (IS_ERR(sai->clk_x11k)) { - dev_err(&pdev->dev, "missing x11k parent clock\n"); + if (PTR_ERR(sai->clk_x11k) != -EPROBE_DEFER) + dev_err(&pdev->dev, "missing x11k parent clock: %ld\n", + PTR_ERR(sai->clk_x11k)); return PTR_ERR(sai->clk_x11k); } @@ -197,12 +203,16 @@ static int stm32_sai_probe(struct platform_device *pdev) return sai->irq; /* reset */ - rst = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (!IS_ERR(rst)) { - reset_control_assert(rst); - udelay(2); - reset_control_deassert(rst); + rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); + if (IS_ERR(rst)) { + if (PTR_ERR(rst) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Reset controller error %ld\n", + PTR_ERR(rst)); + return PTR_ERR(rst); } + reset_control_assert(rst); + udelay(2); + reset_control_deassert(rst); /* Enable peripheral clock to allow register access */ ret = clk_prepare_enable(sai->pclk); diff --git a/sound/soc/stm/stm32_sai_sub.c b/sound/soc/stm/stm32_sai_sub.c index 10eb4b8e8e7e..2bd280c01c33 100644 --- a/sound/soc/stm/stm32_sai_sub.c +++ b/sound/soc/stm/stm32_sai_sub.c @@ -1238,7 +1238,7 @@ static int stm32_sai_pcm_process_spdif(struct snd_pcm_substream *substream, { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct stm32_sai_sub_data *sai = dev_get_drvdata(cpu_dai->dev); int *ptr = (int *)(runtime->dma_area + hwoff + channel * (runtime->dma_bytes / runtime->channels)); @@ -1380,7 +1380,9 @@ static int stm32_sai_sub_parse_of(struct platform_device *pdev, sai->regmap = devm_regmap_init_mmio(&pdev->dev, base, sai->regmap_config); if (IS_ERR(sai->regmap)) { - dev_err(&pdev->dev, "Failed to initialize MMIO\n"); + if (PTR_ERR(sai->regmap) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Regmap init error %ld\n", + PTR_ERR(sai->regmap)); return PTR_ERR(sai->regmap); } @@ -1471,7 +1473,9 @@ static int stm32_sai_sub_parse_of(struct platform_device *pdev, of_node_put(args.np); sai->sai_ck = devm_clk_get(&pdev->dev, "sai_ck"); if (IS_ERR(sai->sai_ck)) { - dev_err(&pdev->dev, "Missing kernel clock sai_ck\n"); + if (PTR_ERR(sai->sai_ck) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Missing kernel clock sai_ck: %ld\n", + PTR_ERR(sai->sai_ck)); return PTR_ERR(sai->sai_ck); } @@ -1545,7 +1549,8 @@ static int stm32_sai_sub_probe(struct platform_device *pdev) ret = snd_dmaengine_pcm_register(&pdev->dev, conf, 0); if (ret) { - dev_err(&pdev->dev, "Could not register pcm dma\n"); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Could not register pcm dma\n"); return ret; } diff --git a/sound/soc/stm/stm32_spdifrx.c b/sound/soc/stm/stm32_spdifrx.c index 3769d9ce5dbe..1bfa3b2ba974 100644 --- a/sound/soc/stm/stm32_spdifrx.c +++ b/sound/soc/stm/stm32_spdifrx.c @@ -406,7 +406,9 @@ static int stm32_spdifrx_dma_ctrl_register(struct device *dev, spdifrx->ctrl_chan = dma_request_chan(dev, "rx-ctrl"); if (IS_ERR(spdifrx->ctrl_chan)) { - dev_err(dev, "dma_request_slave_channel failed\n"); + if (PTR_ERR(spdifrx->ctrl_chan) != -EPROBE_DEFER) + dev_err(dev, "dma_request_slave_channel error %ld\n", + PTR_ERR(spdifrx->ctrl_chan)); return PTR_ERR(spdifrx->ctrl_chan); } @@ -929,7 +931,9 @@ static int stm32_spdifrx_parse_of(struct platform_device *pdev, spdifrx->kclk = devm_clk_get(&pdev->dev, "kclk"); if (IS_ERR(spdifrx->kclk)) { - dev_err(&pdev->dev, "Could not get kclk\n"); + if (PTR_ERR(spdifrx->kclk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Could not get kclk: %ld\n", + PTR_ERR(spdifrx->kclk)); return PTR_ERR(spdifrx->kclk); } @@ -940,6 +944,22 @@ static int stm32_spdifrx_parse_of(struct platform_device *pdev, return 0; } +static int stm32_spdifrx_remove(struct platform_device *pdev) +{ + struct stm32_spdifrx_data *spdifrx = platform_get_drvdata(pdev); + + if (spdifrx->ctrl_chan) + dma_release_channel(spdifrx->ctrl_chan); + + if (spdifrx->dmab) + snd_dma_free_pages(spdifrx->dmab); + + snd_dmaengine_pcm_unregister(&pdev->dev); + snd_soc_unregister_component(&pdev->dev); + + return 0; +} + static int stm32_spdifrx_probe(struct platform_device *pdev) { struct stm32_spdifrx_data *spdifrx; @@ -967,7 +987,9 @@ static int stm32_spdifrx_probe(struct platform_device *pdev) spdifrx->base, spdifrx->regmap_conf); if (IS_ERR(spdifrx->regmap)) { - dev_err(&pdev->dev, "Regmap init failed\n"); + if (PTR_ERR(spdifrx->regmap) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Regmap init error %ld\n", + PTR_ERR(spdifrx->regmap)); return PTR_ERR(spdifrx->regmap); } @@ -978,37 +1000,46 @@ static int stm32_spdifrx_probe(struct platform_device *pdev) return ret; } - rst = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (!IS_ERR(rst)) { - reset_control_assert(rst); - udelay(2); - reset_control_deassert(rst); + rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); + if (IS_ERR(rst)) { + if (PTR_ERR(rst) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Reset controller error %ld\n", + PTR_ERR(rst)); + return PTR_ERR(rst); + } + reset_control_assert(rst); + udelay(2); + reset_control_deassert(rst); + + pcm_config = &stm32_spdifrx_pcm_config; + ret = snd_dmaengine_pcm_register(&pdev->dev, pcm_config, 0); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "PCM DMA register error %d\n", ret); + return ret; } - ret = devm_snd_soc_register_component(&pdev->dev, - &stm32_spdifrx_component, - stm32_spdifrx_dai, - ARRAY_SIZE(stm32_spdifrx_dai)); - if (ret) + ret = snd_soc_register_component(&pdev->dev, + &stm32_spdifrx_component, + stm32_spdifrx_dai, + ARRAY_SIZE(stm32_spdifrx_dai)); + if (ret) { + snd_dmaengine_pcm_unregister(&pdev->dev); return ret; + } ret = stm32_spdifrx_dma_ctrl_register(&pdev->dev, spdifrx); if (ret) goto error; - pcm_config = &stm32_spdifrx_pcm_config; - ret = devm_snd_dmaengine_pcm_register(&pdev->dev, pcm_config, 0); - if (ret) { - dev_err(&pdev->dev, "PCM DMA register returned %d\n", ret); - goto error; - } - ret = regmap_read(spdifrx->regmap, STM32_SPDIFRX_IDR, &idr); if (ret) goto error; if (idr == SPDIFRX_IPIDR_NUMBER) { ret = regmap_read(spdifrx->regmap, STM32_SPDIFRX_VERR, &ver); + if (ret) + goto error; dev_dbg(&pdev->dev, "SPDIFRX version: %lu.%lu registered\n", FIELD_GET(SPDIFRX_VERR_MAJ_MASK, ver), @@ -1018,27 +1049,11 @@ static int stm32_spdifrx_probe(struct platform_device *pdev) return ret; error: - if (!IS_ERR(spdifrx->ctrl_chan)) - dma_release_channel(spdifrx->ctrl_chan); - if (spdifrx->dmab) - snd_dma_free_pages(spdifrx->dmab); + stm32_spdifrx_remove(pdev); return ret; } -static int stm32_spdifrx_remove(struct platform_device *pdev) -{ - struct stm32_spdifrx_data *spdifrx = platform_get_drvdata(pdev); - - if (spdifrx->ctrl_chan) - dma_release_channel(spdifrx->ctrl_chan); - - if (spdifrx->dmab) - snd_dma_free_pages(spdifrx->dmab); - - return 0; -} - MODULE_DEVICE_TABLE(of, stm32_spdifrx_ids); #ifdef CONFIG_PM_SLEEP diff --git a/sound/soc/sunxi/sun4i-spdif.c b/sound/soc/sunxi/sun4i-spdif.c index 98a9fe645521..86779a99df75 100644 --- a/sound/soc/sunxi/sun4i-spdif.c +++ b/sound/soc/sunxi/sun4i-spdif.c @@ -244,7 +244,7 @@ static int sun4i_spdif_startup(struct snd_pcm_substream *substream, struct snd_soc_dai *cpu_dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct sun4i_spdif_dev *host = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct sun4i_spdif_dev *host = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); if (substream->stream != SNDRV_PCM_STREAM_PLAYBACK) return -EINVAL; diff --git a/sound/soc/sunxi/sun8i-codec.c b/sound/soc/sunxi/sun8i-codec.c index 686561df8e13..ca51af114419 100644 --- a/sound/soc/sunxi/sun8i-codec.c +++ b/sound/soc/sunxi/sun8i-codec.c @@ -86,7 +86,6 @@ #define SUN8I_AIF1CLK_CTRL_AIF1_BCLK_DIV_MASK GENMASK(12, 9) struct sun8i_codec { - struct device *dev; struct regmap *regmap; struct clk *clk_module; struct clk *clk_bus; @@ -542,8 +541,6 @@ static int sun8i_codec_probe(struct platform_device *pdev) if (!scodec) return -ENOMEM; - scodec->dev = &pdev->dev; - scodec->clk_module = devm_clk_get(&pdev->dev, "mod"); if (IS_ERR(scodec->clk_module)) { dev_err(&pdev->dev, "Failed to get the module clock\n"); diff --git a/sound/soc/tegra/tegra_alc5632.c b/sound/soc/tegra/tegra_alc5632.c index 9e8b1497efd3..ec39ecba1e8b 100644 --- a/sound/soc/tegra/tegra_alc5632.c +++ b/sound/soc/tegra/tegra_alc5632.c @@ -37,7 +37,7 @@ static int tegra_alc5632_asoc_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct tegra_alc5632 *alc5632 = snd_soc_card_get_drvdata(card); int srate, mclk; diff --git a/sound/soc/tegra/tegra_max98090.c b/sound/soc/tegra/tegra_max98090.c index 4954a33ff46b..d800b62b36f8 100644 --- a/sound/soc/tegra/tegra_max98090.c +++ b/sound/soc/tegra/tegra_max98090.c @@ -38,7 +38,7 @@ static int tegra_max98090_asoc_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct tegra_max98090 *machine = snd_soc_card_get_drvdata(card); int srate, mclk; diff --git a/sound/soc/tegra/tegra_rt5640.c b/sound/soc/tegra/tegra_rt5640.c index d46915a3ec4c..9878bc3eb89e 100644 --- a/sound/soc/tegra/tegra_rt5640.c +++ b/sound/soc/tegra/tegra_rt5640.c @@ -40,7 +40,7 @@ static int tegra_rt5640_asoc_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct tegra_rt5640 *machine = snd_soc_card_get_drvdata(card); int srate, mclk; diff --git a/sound/soc/tegra/tegra_rt5677.c b/sound/soc/tegra/tegra_rt5677.c index 81cb6cc6236e..5821313db977 100644 --- a/sound/soc/tegra/tegra_rt5677.c +++ b/sound/soc/tegra/tegra_rt5677.c @@ -42,7 +42,7 @@ static int tegra_rt5677_asoc_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct tegra_rt5677 *machine = snd_soc_card_get_drvdata(card); int srate, mclk, err; diff --git a/sound/soc/tegra/tegra_sgtl5000.c b/sound/soc/tegra/tegra_sgtl5000.c index e13b81d29cf3..dc411ba2e36d 100644 --- a/sound/soc/tegra/tegra_sgtl5000.c +++ b/sound/soc/tegra/tegra_sgtl5000.c @@ -36,7 +36,7 @@ static int tegra_sgtl5000_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct tegra_sgtl5000 *machine = snd_soc_card_get_drvdata(card); int srate, mclk; diff --git a/sound/soc/tegra/tegra_wm8753.c b/sound/soc/tegra/tegra_wm8753.c index f6dd790dad71..0d653a605358 100644 --- a/sound/soc/tegra/tegra_wm8753.c +++ b/sound/soc/tegra/tegra_wm8753.c @@ -40,7 +40,7 @@ static int tegra_wm8753_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct tegra_wm8753 *machine = snd_soc_card_get_drvdata(card); int srate, mclk; diff --git a/sound/soc/tegra/tegra_wm8903.c b/sound/soc/tegra/tegra_wm8903.c index f08d3489c3cf..9b5651502f12 100644 --- a/sound/soc/tegra/tegra_wm8903.c +++ b/sound/soc/tegra/tegra_wm8903.c @@ -45,7 +45,7 @@ static int tegra_wm8903_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct tegra_wm8903 *machine = snd_soc_card_get_drvdata(card); int srate, mclk; @@ -143,19 +143,37 @@ static int tegra_wm8903_event_hp(struct snd_soc_dapm_widget *w, return 0; } +static int tegra_wm8903_event_int_mic(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *k, int event) +{ + struct snd_soc_dapm_context *dapm = w->dapm; + struct snd_soc_card *card = dapm->card; + struct tegra_wm8903 *machine = snd_soc_card_get_drvdata(card); + + if (!gpio_is_valid(machine->gpio_int_mic_en)) + return 0; + + gpio_set_value_cansleep(machine->gpio_int_mic_en, + SND_SOC_DAPM_EVENT_ON(event)); + + return 0; +} + static const struct snd_soc_dapm_widget tegra_wm8903_dapm_widgets[] = { SND_SOC_DAPM_SPK("Int Spk", tegra_wm8903_event_int_spk), SND_SOC_DAPM_HP("Headphone Jack", tegra_wm8903_event_hp), SND_SOC_DAPM_MIC("Mic Jack", NULL), + SND_SOC_DAPM_MIC("Int Mic", tegra_wm8903_event_int_mic), }; static const struct snd_kcontrol_new tegra_wm8903_controls[] = { SOC_DAPM_PIN_SWITCH("Int Spk"), + SOC_DAPM_PIN_SWITCH("Int Mic"), }; static int tegra_wm8903_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_component *component = codec_dai->component; struct snd_soc_card *card = rtd->card; struct tegra_wm8903 *machine = snd_soc_card_get_drvdata(card); @@ -187,7 +205,7 @@ static int tegra_wm8903_remove(struct snd_soc_card *card) { struct snd_soc_pcm_runtime *rtd = snd_soc_get_pcm_runtime(card, &card->dai_link[0]); - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_component *component = codec_dai->component; wm8903_mic_detect(component, NULL, 0, 0); diff --git a/sound/soc/tegra/trimslice.c b/sound/soc/tegra/trimslice.c index 3f67ddd13674..f9834afaa2e8 100644 --- a/sound/soc/tegra/trimslice.c +++ b/sound/soc/tegra/trimslice.c @@ -35,7 +35,7 @@ static int trimslice_asoc_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct tegra_trimslice *trimslice = snd_soc_card_get_drvdata(card); int srate, mclk; diff --git a/sound/soc/ti/Kconfig b/sound/soc/ti/Kconfig index 29f61053ab62..c5408c129f34 100644 --- a/sound/soc/ti/Kconfig +++ b/sound/soc/ti/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only menu "Audio support for Texas Instruments SoCs" -depends on DMA_OMAP || TI_EDMA || COMPILE_TEST +depends on DMA_OMAP || TI_EDMA || TI_K3_UDMA || COMPILE_TEST config SND_SOC_TI_EDMA_PCM tristate @@ -10,6 +10,10 @@ config SND_SOC_TI_SDMA_PCM tristate select SND_SOC_GENERIC_DMAENGINE_PCM +config SND_SOC_TI_UDMA_PCM + tristate + select SND_SOC_GENERIC_DMAENGINE_PCM + comment "Texas Instruments DAI support for:" config SND_SOC_DAVINCI_ASP tristate "daVinci Audio Serial Port (ASP) or McBSP support" @@ -24,6 +28,7 @@ config SND_SOC_DAVINCI_MCASP tristate "Multichannel Audio Serial Port (McASP) support" select SND_SOC_TI_EDMA_PCM select SND_SOC_TI_SDMA_PCM + select SND_SOC_TI_UDMA_PCM help Say Y or M here if you want to have support for McASP IP found in various Texas Instruments SoCs like: @@ -31,6 +36,7 @@ config SND_SOC_DAVINCI_MCASP - Sitara line of SoCs (AM335x, AM438x, etc) - DRA7x devices - Keystone devices + - K3 devices (am654, j721e) config SND_SOC_DAVINCI_VCIF tristate "daVinci Voice Interface (VCIF) support" diff --git a/sound/soc/ti/Makefile b/sound/soc/ti/Makefile index 08c44d56ef3e..ea48c6679cc7 100644 --- a/sound/soc/ti/Makefile +++ b/sound/soc/ti/Makefile @@ -3,9 +3,11 @@ # Platform drivers snd-soc-ti-edma-objs := edma-pcm.o snd-soc-ti-sdma-objs := sdma-pcm.o +snd-soc-ti-udma-objs := udma-pcm.o obj-$(CONFIG_SND_SOC_TI_EDMA_PCM) += snd-soc-ti-edma.o obj-$(CONFIG_SND_SOC_TI_SDMA_PCM) += snd-soc-ti-sdma.o +obj-$(CONFIG_SND_SOC_TI_UDMA_PCM) += snd-soc-ti-udma.o # CPU DAI drivers snd-soc-davinci-asp-objs := davinci-i2s.o diff --git a/sound/soc/ti/ams-delta.c b/sound/soc/ti/ams-delta.c index 8e2fb81ad05c..e17cd5e939f0 100644 --- a/sound/soc/ti/ams-delta.c +++ b/sound/soc/ti/ams-delta.c @@ -460,14 +460,14 @@ static void ams_delta_shutdown(struct snd_pcm_substream *substream) static int ams_delta_cx20442_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct snd_soc_dapm_context *dapm = &card->dapm; int ret; /* Codec is ready, now add/activate board specific controls */ /* Store a pointer to the codec structure for tty ldisc use */ - cx20442_codec = rtd->codec_dai->component; + cx20442_codec = asoc_rtd_to_codec(rtd, 0)->component; /* Add hook switch - can be used to control the codec from userspace * even if line discipline fails */ diff --git a/sound/soc/ti/davinci-evm.c b/sound/soc/ti/davinci-evm.c index 686b23d7a90d..2cfbeebdfb41 100644 --- a/sound/soc/ti/davinci-evm.c +++ b/sound/soc/ti/davinci-evm.c @@ -54,8 +54,8 @@ static int evm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct snd_soc_card *soc_card = rtd->card; int ret = 0; unsigned sysclk = ((struct snd_soc_card_drvdata_davinci *) diff --git a/sound/soc/ti/davinci-mcasp.c b/sound/soc/ti/davinci-mcasp.c index e1e937eb1dc1..734ffe925c4d 100644 --- a/sound/soc/ti/davinci-mcasp.c +++ b/sound/soc/ti/davinci-mcasp.c @@ -38,6 +38,7 @@ #include "edma-pcm.h" #include "sdma-pcm.h" +#include "udma-pcm.h" #include "davinci-mcasp.h" #define MCASP_MAX_AFIFO_DEPTH 64 @@ -1764,10 +1765,8 @@ static struct davinci_mcasp_pdata *davinci_mcasp_set_pdata_from_of( } else if (match) { pdata = devm_kmemdup(&pdev->dev, match->data, sizeof(*pdata), GFP_KERNEL); - if (!pdata) { - ret = -ENOMEM; - return pdata; - } + if (!pdata) + return NULL; } else { /* control shouldn't reach here. something is wrong */ ret = -EINVAL; @@ -1875,6 +1874,7 @@ static struct davinci_mcasp_pdata *davinci_mcasp_set_pdata_from_of( enum { PCM_EDMA, PCM_SDMA, + PCM_UDMA, }; static const char *sdma_prefix = "ti,omap"; @@ -1912,6 +1912,8 @@ static int davinci_mcasp_get_dma_type(struct davinci_mcasp *mcasp) dev_dbg(mcasp->dev, "DMA controller compatible = \"%s\"\n", tmp); if (!strncmp(tmp, sdma_prefix, strlen(sdma_prefix))) return PCM_SDMA; + else if (strstr(tmp, "udmap")) + return PCM_UDMA; return PCM_EDMA; } @@ -2371,6 +2373,9 @@ static int davinci_mcasp_probe(struct platform_device *pdev) case PCM_SDMA: ret = sdma_pcm_platform_register(&pdev->dev, "tx", "rx"); break; + case PCM_UDMA: + ret = udma_pcm_platform_register(&pdev->dev); + break; default: dev_err(&pdev->dev, "No DMA controller found (%d)\n", ret); case -EPROBE_DEFER: diff --git a/sound/soc/ti/davinci-vcif.c b/sound/soc/ti/davinci-vcif.c index c84650e4a7aa..ee4d3ef821a1 100644 --- a/sound/soc/ti/davinci-vcif.c +++ b/sound/soc/ti/davinci-vcif.c @@ -43,7 +43,7 @@ static void davinci_vcif_start(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct davinci_vcif_dev *davinci_vcif_dev = - snd_soc_dai_get_drvdata(rtd->cpu_dai); + snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct davinci_vc *davinci_vc = davinci_vcif_dev->davinci_vc; u32 w; @@ -62,7 +62,7 @@ static void davinci_vcif_stop(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct davinci_vcif_dev *davinci_vcif_dev = - snd_soc_dai_get_drvdata(rtd->cpu_dai); + snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); struct davinci_vc *davinci_vc = davinci_vcif_dev->davinci_vc; u32 w; diff --git a/sound/soc/ti/n810.c b/sound/soc/ti/n810.c index 3ad2b6daf31e..a1672b479cb7 100644 --- a/sound/soc/ti/n810.c +++ b/sound/soc/ti/n810.c @@ -101,7 +101,7 @@ static int n810_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int err; /* Set the codec system clock for DAC and ADC */ diff --git a/sound/soc/ti/omap-abe-twl6040.c b/sound/soc/ti/omap-abe-twl6040.c index 6d564ab5e437..61e45fea5dd8 100644 --- a/sound/soc/ti/omap-abe-twl6040.c +++ b/sound/soc/ti/omap-abe-twl6040.c @@ -46,7 +46,7 @@ static int omap_abe_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); struct snd_soc_card *card = rtd->card; struct abe_twl6040 *priv = snd_soc_card_get_drvdata(card); int clk_id, freq; @@ -78,7 +78,7 @@ static int omap_abe_dmic_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int ret = 0; ret = snd_soc_dai_set_sysclk(cpu_dai, OMAP_DMIC_SYSCLK_PAD_CLKS, @@ -166,7 +166,7 @@ static const struct snd_soc_dapm_route audio_map[] = { static int omap_abe_twl6040_init(struct snd_soc_pcm_runtime *rtd) { - struct snd_soc_component *component = rtd->codec_dai->component; + struct snd_soc_component *component = asoc_rtd_to_codec(rtd, 0)->component; struct snd_soc_card *card = rtd->card; struct abe_twl6040 *priv = snd_soc_card_get_drvdata(card); int hs_trim; diff --git a/sound/soc/ti/omap-mcbsp-st.c b/sound/soc/ti/omap-mcbsp-st.c index 1a3fe854e856..5a32b54bbf3b 100644 --- a/sound/soc/ti/omap-mcbsp-st.c +++ b/sound/soc/ti/omap-mcbsp-st.c @@ -489,7 +489,7 @@ OMAP_MCBSP_ST_CONTROLS(3); int omap_mcbsp_st_add_controls(struct snd_soc_pcm_runtime *rtd, int port_id) { - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai); if (!mcbsp->st_data) { diff --git a/sound/soc/ti/omap-mcbsp.c b/sound/soc/ti/omap-mcbsp.c index 302d5c493c29..3d41ca2238d4 100644 --- a/sound/soc/ti/omap-mcbsp.c +++ b/sound/soc/ti/omap-mcbsp.c @@ -737,7 +737,7 @@ static void omap_mcbsp_set_threshold(struct snd_pcm_substream *substream, unsigned int packet_size) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai); int words; @@ -902,7 +902,7 @@ static snd_pcm_sframes_t omap_mcbsp_dai_delay( struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct omap_mcbsp *mcbsp = snd_soc_dai_get_drvdata(cpu_dai); u16 fifo_use; snd_pcm_sframes_t delay; diff --git a/sound/soc/ti/omap-mcpdm.c b/sound/soc/ti/omap-mcpdm.c index d7ac4df6f2d9..f2dbadea33bb 100644 --- a/sound/soc/ti/omap-mcpdm.c +++ b/sound/soc/ti/omap-mcpdm.c @@ -532,7 +532,7 @@ static const struct snd_soc_component_driver omap_mcpdm_component = { void omap_mcpdm_configure_dn_offsets(struct snd_soc_pcm_runtime *rtd, u8 rx1, u8 rx2) { - struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(rtd->cpu_dai); + struct omap_mcpdm *mcpdm = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); mcpdm->dn_rx_offset = MCPDM_DNOFST_RX1(rx1) | MCPDM_DNOFST_RX2(rx2); } diff --git a/sound/soc/ti/omap3pandora.c b/sound/soc/ti/omap3pandora.c index 545f8dac9bd5..b04146311b31 100644 --- a/sound/soc/ti/omap3pandora.c +++ b/sound/soc/ti/omap3pandora.c @@ -32,8 +32,8 @@ static int omap3pandora_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); int ret; /* Set the codec system clock for DAC and ADC */ diff --git a/sound/soc/ti/osk5912.c b/sound/soc/ti/osk5912.c index 1ca466bc4025..e01485cc51a1 100644 --- a/sound/soc/ti/osk5912.c +++ b/sound/soc/ti/osk5912.c @@ -39,7 +39,7 @@ static int osk_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); int err; /* Set the codec system clock for DAC and ADC */ diff --git a/sound/soc/ti/rx51.c b/sound/soc/ti/rx51.c index fdb0dc85fe67..2a714a004163 100644 --- a/sound/soc/ti/rx51.c +++ b/sound/soc/ti/rx51.c @@ -103,7 +103,7 @@ static int rx51_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); /* Set the codec system clock for DAC and ADC */ return snd_soc_dai_set_sysclk(codec_dai, 0, 19200000, diff --git a/sound/soc/ti/udma-pcm.c b/sound/soc/ti/udma-pcm.c new file mode 100644 index 000000000000..39830caaaf7c --- /dev/null +++ b/sound/soc/ti/udma-pcm.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com + * Author: Peter Ujfalusi + */ + +#include +#include +#include +#include +#include +#include + +#include "udma-pcm.h" + +static const struct snd_pcm_hardware udma_pcm_hardware = { + .info = SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME | + SNDRV_PCM_INFO_NO_PERIOD_WAKEUP | + SNDRV_PCM_INFO_INTERLEAVED, + .buffer_bytes_max = SIZE_MAX, + .period_bytes_min = 32, + .period_bytes_max = SZ_64K, + .periods_min = 2, + .periods_max = UINT_MAX, +}; + +static const struct snd_dmaengine_pcm_config udma_dmaengine_pcm_config = { + .pcm_hardware = &udma_pcm_hardware, + .prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config, +}; + +int udma_pcm_platform_register(struct device *dev) +{ + return devm_snd_dmaengine_pcm_register(dev, &udma_dmaengine_pcm_config, + 0); +} +EXPORT_SYMBOL_GPL(udma_pcm_platform_register); + +MODULE_AUTHOR("Peter Ujfalusi "); +MODULE_DESCRIPTION("UDMA PCM ASoC platform driver"); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/ti/udma-pcm.h b/sound/soc/ti/udma-pcm.h new file mode 100644 index 000000000000..54111e7312c1 --- /dev/null +++ b/sound/soc/ti/udma-pcm.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com + */ + +#ifndef __UDMA_PCM_H__ +#define __UDMA_PCM_H__ + +#if IS_ENABLED(CONFIG_SND_SOC_TI_UDMA_PCM) +int udma_pcm_platform_register(struct device *dev); +#else +static inline int udma_pcm_platform_register(struct device *dev) +{ + return 0; +} +#endif /* CONFIG_SND_SOC_TI_UDMA_PCM */ + +#endif /* __UDMA_PCM_H__ */ diff --git a/sound/soc/txx9/txx9aclc.c b/sound/soc/txx9/txx9aclc.c index 985487cc3a55..4b1cd4da3e36 100644 --- a/sound/soc/txx9/txx9aclc.c +++ b/sound/soc/txx9/txx9aclc.c @@ -269,7 +269,7 @@ static int txx9aclc_pcm_new(struct snd_soc_component *component, struct snd_soc_pcm_runtime *rtd) { struct snd_card *card = rtd->card->snd_card; - struct snd_soc_dai *dai = rtd->cpu_dai; + struct snd_soc_dai *dai = asoc_rtd_to_cpu(rtd, 0); struct snd_pcm *pcm = rtd->pcm; struct platform_device *pdev = to_platform_device(component->dev); struct txx9aclc_soc_device *dev; diff --git a/sound/soc/uniphier/aio-compress.c b/sound/soc/uniphier/aio-compress.c index 17f773ac5ca1..232d3cc5bce0 100644 --- a/sound/soc/uniphier/aio-compress.c +++ b/sound/soc/uniphier/aio-compress.c @@ -23,7 +23,7 @@ static int uniphier_aio_comprdma_new(struct snd_soc_pcm_runtime *rtd) { struct snd_compr *compr = rtd->compr; struct device *dev = compr->card->dev; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[compr->direction]; size_t size = AUD_RING_SIZE; int dma_dir = DMA_FROM_DEVICE, ret; @@ -56,7 +56,7 @@ static int uniphier_aio_comprdma_free(struct snd_soc_pcm_runtime *rtd) { struct snd_compr *compr = rtd->compr; struct device *dev = compr->card->dev; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[compr->direction]; int dma_dir = DMA_FROM_DEVICE; @@ -73,7 +73,7 @@ static int uniphier_aio_comprdma_free(struct snd_soc_pcm_runtime *rtd) static int uniphier_aio_compr_open(struct snd_compr_stream *cstream) { struct snd_soc_pcm_runtime *rtd = cstream->private_data; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; int ret; @@ -98,7 +98,7 @@ static int uniphier_aio_compr_open(struct snd_compr_stream *cstream) static int uniphier_aio_compr_free(struct snd_compr_stream *cstream) { struct snd_soc_pcm_runtime *rtd = cstream->private_data; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; int ret; @@ -118,7 +118,7 @@ static int uniphier_aio_compr_get_params(struct snd_compr_stream *cstream, struct snd_codec *params) { struct snd_soc_pcm_runtime *rtd = cstream->private_data; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; *params = sub->cparams.codec; @@ -130,7 +130,7 @@ static int uniphier_aio_compr_set_params(struct snd_compr_stream *cstream, struct snd_compr_params *params) { struct snd_soc_pcm_runtime *rtd = cstream->private_data; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; struct device *dev = &aio->chip->pdev->dev; int ret; @@ -165,7 +165,7 @@ static int uniphier_aio_compr_set_params(struct snd_compr_stream *cstream, static int uniphier_aio_compr_hw_free(struct snd_compr_stream *cstream) { struct snd_soc_pcm_runtime *rtd = cstream->private_data; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; sub->setting = 0; @@ -177,7 +177,7 @@ static int uniphier_aio_compr_prepare(struct snd_compr_stream *cstream) { struct snd_soc_pcm_runtime *rtd = cstream->private_data; struct snd_compr_runtime *runtime = cstream->runtime; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; int bytes = runtime->fragment_size; unsigned long flags; @@ -215,7 +215,7 @@ static int uniphier_aio_compr_trigger(struct snd_compr_stream *cstream, { struct snd_soc_pcm_runtime *rtd = cstream->private_data; struct snd_compr_runtime *runtime = cstream->runtime; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; struct device *dev = &aio->chip->pdev->dev; int bytes = runtime->fragment_size, ret = 0; @@ -248,7 +248,7 @@ static int uniphier_aio_compr_pointer(struct snd_compr_stream *cstream, { struct snd_soc_pcm_runtime *rtd = cstream->private_data; struct snd_compr_runtime *runtime = cstream->runtime; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; int bytes = runtime->fragment_size; unsigned long flags; @@ -322,7 +322,7 @@ static int uniphier_aio_compr_copy(struct snd_compr_stream *cstream, struct snd_soc_pcm_runtime *rtd = cstream->private_data; struct snd_compr_runtime *runtime = cstream->runtime; struct device *carddev = rtd->compr->card->dev; - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; size_t cnt = min_t(size_t, count, aio_rb_space_to_end(sub) / 2); int bytes = runtime->fragment_size; diff --git a/sound/soc/uniphier/aio-dma.c b/sound/soc/uniphier/aio-dma.c index da83423c52e2..4bbcb007df41 100644 --- a/sound/soc/uniphier/aio-dma.c +++ b/sound/soc/uniphier/aio-dma.c @@ -109,7 +109,7 @@ static int uniphier_aiodma_prepare(struct snd_soc_component *component, { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *rtd = snd_pcm_substream_chip(substream); - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[substream->stream]; int bytes = runtime->period_size * runtime->channels * samples_to_bytes(runtime, 1); @@ -136,7 +136,7 @@ static int uniphier_aiodma_trigger(struct snd_soc_component *component, { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *rtd = snd_pcm_substream_chip(substream); - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[substream->stream]; struct device *dev = &aio->chip->pdev->dev; int bytes = runtime->period_size * @@ -172,7 +172,7 @@ static snd_pcm_uframes_t uniphier_aiodma_pointer( { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_soc_pcm_runtime *rtd = snd_pcm_substream_chip(substream); - struct uniphier_aio *aio = uniphier_priv(rtd->cpu_dai); + struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[substream->stream]; int bytes = runtime->period_size * runtime->channels * samples_to_bytes(runtime, 1); diff --git a/sound/soc/ux500/mop500_ab8500.c b/sound/soc/ux500/mop500_ab8500.c index 77655084bbde..6aaa19829a73 100644 --- a/sound/soc/ux500/mop500_ab8500.c +++ b/sound/soc/ux500/mop500_ab8500.c @@ -215,8 +215,8 @@ static int mop500_ab8500_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai = rtd->codec_dai; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct device *dev = rtd->card->dev; unsigned int fmt; int channels, ret = 0, driver_mode, slots; @@ -339,7 +339,7 @@ static int mop500_ab8500_hw_params(struct snd_pcm_substream *substream, static int mop500_ab8500_hw_free(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = rtd->cpu_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); mutex_lock(&mop500_ab8500_params_lock); __clear_bit(cpu_dai->id, &mop500_ab8500_usage); diff --git a/sound/soc/ux500/ux500_pcm.c b/sound/soc/ux500/ux500_pcm.c index 9445dbe8e039..39b96c132bc8 100644 --- a/sound/soc/ux500/ux500_pcm.c +++ b/sound/soc/ux500/ux500_pcm.c @@ -46,7 +46,7 @@ static const struct snd_pcm_hardware ux500_pcm_hw = { static struct dma_chan *ux500_pcm_request_chan(struct snd_soc_pcm_runtime *rtd, struct snd_pcm_substream *substream) { - struct snd_soc_dai *dai = rtd->cpu_dai; + struct snd_soc_dai *dai = asoc_rtd_to_cpu(rtd, 0); u16 per_data_width, mem_data_width; struct stedma40_chan_cfg *dma_cfg; struct ux500_msp_dma_params *dma_params; @@ -86,7 +86,7 @@ static int ux500_pcm_prepare_slave_config(struct snd_pcm_substream *substream, struct dma_slave_config *slave_config) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct msp_i2s_platform_data *pdata = rtd->cpu_dai->dev->platform_data; + struct msp_i2s_platform_data *pdata = asoc_rtd_to_cpu(rtd, 0)->dev->platform_data; struct snd_dmaengine_dai_dma_data *snd_dma_params; struct ux500_msp_dma_params *ste_dma_params; dma_addr_t dma_addr; @@ -94,11 +94,11 @@ static int ux500_pcm_prepare_slave_config(struct snd_pcm_substream *substream, if (pdata) { ste_dma_params = - snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); dma_addr = ste_dma_params->tx_rx_addr; } else { snd_dma_params = - snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); dma_addr = snd_dma_params->addr; } diff --git a/sound/soc/xtensa/xtfpga-i2s.c b/sound/soc/xtensa/xtfpga-i2s.c index bcf442faff7c..68af2176b19c 100644 --- a/sound/soc/xtensa/xtfpga-i2s.c +++ b/sound/soc/xtensa/xtfpga-i2s.c @@ -373,7 +373,7 @@ static int xtfpga_pcm_open(struct snd_soc_component *component, void *p; snd_soc_set_runtime_hwparams(substream, &xtfpga_pcm_hardware); - p = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream); + p = snd_soc_dai_get_dma_data(asoc_rtd_to_cpu(rtd, 0), substream); runtime->private_data = p; return 0; diff --git a/sound/soc/zte/zx-spdif.c b/sound/soc/zte/zx-spdif.c index 60382ec23832..a3a07c0730e6 100644 --- a/sound/soc/zte/zx-spdif.c +++ b/sound/soc/zte/zx-spdif.c @@ -322,7 +322,6 @@ static int zx_spdif_probe(struct platform_device *pdev) zx_spdif->mapbase = res->start; zx_spdif->reg_base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(zx_spdif->reg_base)) { - dev_err(&pdev->dev, "ioremap failed!\n"); return PTR_ERR(zx_spdif->reg_base); } diff --git a/sound/soc/zte/zx-tdm.c b/sound/soc/zte/zx-tdm.c index 0e5a05b25a77..4f787185d630 100644 --- a/sound/soc/zte/zx-tdm.c +++ b/sound/soc/zte/zx-tdm.c @@ -371,7 +371,6 @@ static struct snd_soc_dai_driver zx_tdm_dai = { static int zx_tdm_probe(struct platform_device *pdev) { - struct device *dev = &pdev->dev; struct of_phandle_args out_args; unsigned int dma_reg_offset; struct zx_tdm_info *zx_tdm; @@ -384,7 +383,7 @@ static int zx_tdm_probe(struct platform_device *pdev) if (!zx_tdm) return -ENOMEM; - zx_tdm->dev = dev; + zx_tdm->dev = &pdev->dev; zx_tdm->dai_wclk = devm_clk_get(&pdev->dev, "wclk"); if (IS_ERR(zx_tdm->dai_wclk)) { diff --git a/sound/usb/Makefile b/sound/usb/Makefile index 78edd7d2f418..56031026b113 100644 --- a/sound/usb/Makefile +++ b/sound/usb/Makefile @@ -13,6 +13,7 @@ snd-usb-audio-objs := card.o \ mixer_scarlett.o \ mixer_scarlett_gen2.o \ mixer_us16x08.o \ + mixer_s1810c.o \ pcm.o \ power.o \ proc.o \ diff --git a/sound/usb/card.c b/sound/usb/card.c index 5c0b1fb9d237..ec93cde0d1c6 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -72,6 +72,7 @@ static int device_setup[SNDRV_CARDS]; /* device parameter for this card */ static bool ignore_ctl_error; static bool autoclock = true; static char *quirk_alias[SNDRV_CARDS]; +static char *delayed_register[SNDRV_CARDS]; bool snd_usb_use_vmalloc = true; bool snd_usb_skip_validation; @@ -95,6 +96,8 @@ module_param(autoclock, bool, 0444); MODULE_PARM_DESC(autoclock, "Enable auto-clock selection for UAC2 devices (default: yes)."); module_param_array(quirk_alias, charp, NULL, 0444); MODULE_PARM_DESC(quirk_alias, "Quirk aliases, e.g. 0123abcd:5678beef."); +module_param_array(delayed_register, charp, NULL, 0444); +MODULE_PARM_DESC(delayed_register, "Quirk for delayed registration, given by id:iface, e.g. 0123abcd:4."); module_param_named(use_vmalloc, snd_usb_use_vmalloc, bool, 0444); MODULE_PARM_DESC(use_vmalloc, "Use vmalloc for PCM intermediate buffers (default: yes)."); module_param_named(skip_validation, snd_usb_skip_validation, bool, 0444); @@ -593,6 +596,21 @@ static bool get_alias_id(struct usb_device *dev, unsigned int *id) return false; } +static bool check_delayed_register_option(struct snd_usb_audio *chip, int iface) +{ + int i; + unsigned int id, inum; + + for (i = 0; i < ARRAY_SIZE(delayed_register); i++) { + if (delayed_register[i] && + sscanf(delayed_register[i], "%x:%x", &id, &inum) == 2 && + id == chip->usb_id) + return inum != iface; + } + + return false; +} + static const struct usb_device_id usb_audio_ids[]; /* defined below */ /* look for the corresponding quirk */ @@ -730,10 +748,22 @@ static int usb_audio_probe(struct usb_interface *intf, goto __error; } - /* we are allowed to call snd_card_register() many times */ - err = snd_card_register(chip->card); - if (err < 0) - goto __error; + if (chip->need_delayed_register) { + dev_info(&dev->dev, + "Found post-registration device assignment: %08x:%02x\n", + chip->usb_id, ifnum); + chip->need_delayed_register = false; /* clear again */ + } + + /* we are allowed to call snd_card_register() many times, but first + * check to see if a device needs to skip it or do anything special + */ + if (!snd_usb_registration_quirk(chip, ifnum) && + !check_delayed_register_option(chip, ifnum)) { + err = snd_card_register(chip->card); + if (err < 0) + goto __error; + } if (quirk && quirk->shares_media_device) { /* don't want to fail when snd_media_device_create() fails */ diff --git a/sound/usb/clock.c b/sound/usb/clock.c index a48313dfa967..b118cf97607f 100644 --- a/sound/usb/clock.c +++ b/sound/usb/clock.c @@ -151,16 +151,15 @@ static int uac_clock_selector_set_val(struct snd_usb_audio *chip, int selector_i return ret; } -/* - * Assume the clock is valid if clock source supports only one single sample - * rate, the terminal is connected directly to it (there is no clock selector) - * and clock type is internal. This is to deal with some Denon DJ controllers - * that always reports that clock is invalid. - */ static bool uac_clock_source_is_valid_quirk(struct snd_usb_audio *chip, struct audioformat *fmt, int source_id) { + bool ret = false; + int count; + unsigned char data; + struct usb_device *dev = chip->dev; + if (fmt->protocol == UAC_VERSION_2) { struct uac_clock_source_descriptor *cs_desc = snd_usb_find_clock_source(chip->ctrl_intf, source_id); @@ -168,13 +167,51 @@ static bool uac_clock_source_is_valid_quirk(struct snd_usb_audio *chip, if (!cs_desc) return false; - return (fmt->nr_rates == 1 && - (fmt->clock & 0xff) == cs_desc->bClockID && - (cs_desc->bmAttributes & 0x3) != - UAC_CLOCK_SOURCE_TYPE_EXT); + /* + * Assume the clock is valid if clock source supports only one + * single sample rate, the terminal is connected directly to it + * (there is no clock selector) and clock type is internal. + * This is to deal with some Denon DJ controllers that always + * reports that clock is invalid. + */ + if (fmt->nr_rates == 1 && + (fmt->clock & 0xff) == cs_desc->bClockID && + (cs_desc->bmAttributes & 0x3) != + UAC_CLOCK_SOURCE_TYPE_EXT) + return true; } - return false; + /* + * MOTU MicroBook IIc + * Sample rate changes takes more than 2 seconds for this device. Clock + * validity request returns false during that period. + */ + if (chip->usb_id == USB_ID(0x07fd, 0x0004)) { + count = 0; + + while ((!ret) && (count < 50)) { + int err; + + msleep(100); + + err = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), UAC2_CS_CUR, + USB_TYPE_CLASS | USB_RECIP_INTERFACE | USB_DIR_IN, + UAC2_CS_CONTROL_CLOCK_VALID << 8, + snd_usb_ctrl_intf(chip) | (source_id << 8), + &data, sizeof(data)); + if (err < 0) { + dev_warn(&dev->dev, + "%s(): cannot get clock validity for id %d\n", + __func__, source_id); + return false; + } + + ret = !!data; + count++; + } + } + + return ret; } static bool uac_clock_source_is_valid(struct snd_usb_audio *chip, diff --git a/sound/usb/format.c b/sound/usb/format.c index 9f5cb4ed3a0c..50e1874c847c 100644 --- a/sound/usb/format.c +++ b/sound/usb/format.c @@ -247,6 +247,36 @@ static int parse_audio_format_rates_v1(struct snd_usb_audio *chip, struct audiof return 0; } + +/* + * Presonus Studio 1810c supports a limited set of sampling + * rates per altsetting but reports the full set each time. + * If we don't filter out the unsupported rates and attempt + * to configure the card, it will hang refusing to do any + * further audio I/O until a hard reset is performed. + * + * The list of supported rates per altsetting (set of available + * I/O channels) is described in the owner's manual, section 2.2. + */ +static bool s1810c_valid_sample_rate(struct audioformat *fp, + unsigned int rate) +{ + switch (fp->altsetting) { + case 1: + /* All ADAT ports available */ + return rate <= 48000; + case 2: + /* Half of ADAT ports available */ + return (rate == 88200 || rate == 96000); + case 3: + /* Analog I/O only (no S/PDIF nor ADAT) */ + return rate >= 176400; + default: + return false; + } + return false; +} + /* * Helper function to walk the array of sample rate triplets reported by * the device. The problem is that we need to parse whole array first to @@ -283,6 +313,12 @@ static int parse_uac2_sample_rate_range(struct snd_usb_audio *chip, } for (rate = min; rate <= max; rate += res) { + + /* Filter out invalid rates on Presonus Studio 1810c */ + if (chip->usb_id == USB_ID(0x0194f, 0x010c) && + !s1810c_valid_sample_rate(fp, rate)) + goto skip_rate; + if (fp->rate_table) fp->rate_table[nr_rates] = rate; if (!fp->rate_min || rate < fp->rate_min) @@ -297,6 +333,7 @@ static int parse_uac2_sample_rate_range(struct snd_usb_audio *chip, break; } +skip_rate: /* avoid endless loop */ if (res == 0) break; diff --git a/sound/usb/midi.c b/sound/usb/midi.c index 392e5fda680c..047b90595d65 100644 --- a/sound/usb/midi.c +++ b/sound/usb/midi.c @@ -91,7 +91,7 @@ struct usb_ms_endpoint_descriptor { __u8 bDescriptorType; __u8 bDescriptorSubtype; __u8 bNumEmbMIDIJack; - __u8 baAssocJackID[0]; + __u8 baAssocJackID[]; } __attribute__ ((packed)); struct snd_usb_midi_in_endpoint; @@ -1826,6 +1826,28 @@ static int snd_usbmidi_create_endpoints(struct snd_usb_midi *umidi, return 0; } +static struct usb_ms_endpoint_descriptor *find_usb_ms_endpoint_descriptor( + struct usb_host_endpoint *hostep) +{ + unsigned char *extra = hostep->extra; + int extralen = hostep->extralen; + + while (extralen > 3) { + struct usb_ms_endpoint_descriptor *ms_ep = + (struct usb_ms_endpoint_descriptor *)extra; + + if (ms_ep->bLength > 3 && + ms_ep->bDescriptorType == USB_DT_CS_ENDPOINT && + ms_ep->bDescriptorSubtype == UAC_MS_GENERAL) + return ms_ep; + if (!extra[0]) + break; + extralen -= extra[0]; + extra += extra[0]; + } + return NULL; +} + /* * Returns MIDIStreaming device capabilities. */ @@ -1863,11 +1885,8 @@ static int snd_usbmidi_get_ms_info(struct snd_usb_midi *umidi, ep = get_ep_desc(hostep); if (!usb_endpoint_xfer_bulk(ep) && !usb_endpoint_xfer_int(ep)) continue; - ms_ep = (struct usb_ms_endpoint_descriptor *)hostep->extra; - if (hostep->extralen < 4 || - ms_ep->bLength < 4 || - ms_ep->bDescriptorType != USB_DT_CS_ENDPOINT || - ms_ep->bDescriptorSubtype != UAC_MS_GENERAL) + ms_ep = find_usb_ms_endpoint_descriptor(hostep); + if (!ms_ep) continue; if (usb_endpoint_dir_out(ep)) { if (endpoints[epidx].out_ep) { diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index 81b2db0edd5f..721d12130d0c 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -292,6 +292,11 @@ static int uac2_ctl_value_size(int val_type) * retrieve a mixer value */ +static inline int mixer_ctrl_intf(struct usb_mixer_interface *mixer) +{ + return get_iface_desc(mixer->hostif)->bInterfaceNumber; +} + static int get_ctl_value_v1(struct usb_mixer_elem_info *cval, int request, int validx, int *value_ret) { @@ -306,7 +311,7 @@ static int get_ctl_value_v1(struct usb_mixer_elem_info *cval, int request, return -EIO; while (timeout-- > 0) { - idx = snd_usb_ctrl_intf(chip) | (cval->head.id << 8); + idx = mixer_ctrl_intf(cval->head.mixer) | (cval->head.id << 8); err = snd_usb_ctl_msg(chip->dev, usb_rcvctrlpipe(chip->dev, 0), request, USB_RECIP_INTERFACE | USB_TYPE_CLASS | USB_DIR_IN, validx, idx, buf, val_len); @@ -354,7 +359,7 @@ static int get_ctl_value_v2(struct usb_mixer_elem_info *cval, int request, if (ret) goto error; - idx = snd_usb_ctrl_intf(chip) | (cval->head.id << 8); + idx = mixer_ctrl_intf(cval->head.mixer) | (cval->head.id << 8); ret = snd_usb_ctl_msg(chip->dev, usb_rcvctrlpipe(chip->dev, 0), bRequest, USB_RECIP_INTERFACE | USB_TYPE_CLASS | USB_DIR_IN, validx, idx, buf, size); @@ -479,7 +484,7 @@ int snd_usb_mixer_set_ctl_value(struct usb_mixer_elem_info *cval, return -EIO; while (timeout-- > 0) { - idx = snd_usb_ctrl_intf(chip) | (cval->head.id << 8); + idx = mixer_ctrl_intf(cval->head.mixer) | (cval->head.id << 8); err = snd_usb_ctl_msg(chip->dev, usb_sndctrlpipe(chip->dev, 0), request, USB_RECIP_INTERFACE | USB_TYPE_CLASS | USB_DIR_OUT, @@ -901,6 +906,12 @@ static int parse_term_effect_unit(struct mixer_build *state, struct usb_audio_term *term, void *p1, int id) { + struct uac2_effect_unit_descriptor *d = p1; + int err; + + err = __check_input_term(state, d->bSourceID, term); + if (err < 0) + return err; term->type = UAC3_EFFECT_UNIT << 16; /* virtual type */ term->id = id; return 0; @@ -1203,7 +1214,7 @@ static int get_min_max_with_quirks(struct usb_mixer_elem_info *cval, get_ctl_value(cval, UAC_GET_MIN, (cval->control << 8) | minchn, &cval->min) < 0) { usb_audio_err(cval->head.mixer->chip, "%d:%d: cannot get min/max values for control %d (id %d)\n", - cval->head.id, snd_usb_ctrl_intf(cval->head.mixer->chip), + cval->head.id, mixer_ctrl_intf(cval->head.mixer), cval->control, cval->head.id); return -EINVAL; } @@ -1422,7 +1433,7 @@ static int mixer_ctl_connector_get(struct snd_kcontrol *kcontrol, if (ret) goto error; - idx = snd_usb_ctrl_intf(chip) | (cval->head.id << 8); + idx = mixer_ctrl_intf(cval->head.mixer) | (cval->head.id << 8); if (cval->head.mixer->protocol == UAC_VERSION_2) { struct uac2_connectors_ctl_blk uac2_conn; @@ -1674,6 +1685,16 @@ static void __build_feature_ctl(struct usb_mixer_interface *mixer, /* get min/max values */ get_min_max_with_quirks(cval, 0, kctl); + /* skip a bogus volume range */ + if (cval->max <= cval->min) { + usb_audio_dbg(mixer->chip, + "[%d] FU [%s] skipped due to invalid volume\n", + cval->head.id, kctl->id.name); + snd_ctl_free_one(kctl); + return; + } + + if (control == UAC_FU_VOLUME) { check_mapped_dB(map, cval); if (cval->dBmin < cval->dBmax || !cval->initialized) { @@ -3203,7 +3224,7 @@ static void snd_usb_mixer_proc_read(struct snd_info_entry *entry, list_for_each_entry(mixer, &chip->mixer_list, list) { snd_iprintf(buffer, "USB Mixer: usb_id=0x%08x, ctrlif=%i, ctlerr=%i\n", - chip->usb_id, snd_usb_ctrl_intf(chip), + chip->usb_id, mixer_ctrl_intf(mixer), mixer->ignore_ctl_error); snd_iprintf(buffer, "Card: %s\n", chip->card->longname); for (unitid = 0; unitid < MAX_ID_ELEMS; unitid++) { diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c index c237e24f08d9..02b036b2aefb 100644 --- a/sound/usb/mixer_quirks.c +++ b/sound/usb/mixer_quirks.c @@ -34,6 +34,7 @@ #include "mixer_scarlett.h" #include "mixer_scarlett_gen2.h" #include "mixer_us16x08.h" +#include "mixer_s1810c.h" #include "helper.h" struct std_mono_table { @@ -2277,6 +2278,10 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer) case USB_ID(0x2a39, 0x3fd4): /* RME */ err = snd_rme_controls_create(mixer); break; + + case USB_ID(0x0194f, 0x010c): /* Presonus Studio 1810c */ + err = snd_sc1810_init_mixer(mixer); + break; } return err; diff --git a/sound/usb/mixer_s1810c.c b/sound/usb/mixer_s1810c.c new file mode 100644 index 000000000000..6483e47bafd0 --- /dev/null +++ b/sound/usb/mixer_s1810c.c @@ -0,0 +1,595 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Presonus Studio 1810c driver for ALSA + * Copyright (C) 2019 Nick Kossifidis + * + * Based on reverse engineering of the communication protocol + * between the windows driver / Univeral Control (UC) program + * and the device, through usbmon. + * + * For now this bypasses the mixer, with all channels split, + * so that the software can mix with greater flexibility. + * It also adds controls for the 4 buttons on the front of + * the device. + */ + +#include +#include +#include +#include +#include + +#include "usbaudio.h" +#include "mixer.h" +#include "mixer_quirks.h" +#include "helper.h" +#include "mixer_s1810c.h" + +#define SC1810C_CMD_REQ 160 +#define SC1810C_CMD_REQTYPE \ + (USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT) +#define SC1810C_CMD_F1 0x50617269 +#define SC1810C_CMD_F2 0x14 + +/* + * DISCLAIMER: These are just guesses based on the + * dumps I got. + * + * It seems like a selects between + * device (0), mixer (0x64) and output (0x65) + * + * For mixer (0x64): + * * b selects an input channel (see below). + * * c selects an output channel pair (see below). + * * d selects left (0) or right (1) of that pair. + * * e 0-> disconnect, 0x01000000-> connect, + * 0x0109-> used for stereo-linking channels, + * e is also used for setting volume levels + * in which case b is also set so I guess + * this way it is possible to set the volume + * level from the specified input to the + * specified output. + * + * IN Channels: + * 0 - 7 Mic/Inst/Line (Analog inputs) + * 8 - 9 S/PDIF + * 10 - 17 ADAT + * 18 - 35 DAW (Inputs from the host) + * + * OUT Channels (pairs): + * 0 -> Main out + * 1 -> Line1/2 + * 2 -> Line3/4 + * 3 -> S/PDIF + * 4 -> ADAT? + * + * For device (0): + * * b and c are not used, at least not on the + * dumps I got. + * * d sets the control id to be modified + * (see below). + * * e sets the setting for that control. + * (so for the switches I was interested + * in it's 0/1) + * + * For output (0x65): + * * b is the output channel (see above). + * * c is zero. + * * e I guess the same as with mixer except 0x0109 + * which I didn't see in my dumps. + * + * The two fixed fields have the same values for + * mixer and output but a different set for device. + */ +struct s1810c_ctl_packet { + u32 a; + u32 b; + u32 fixed1; + u32 fixed2; + u32 c; + u32 d; + u32 e; +}; + +#define SC1810C_CTL_LINE_SW 0 +#define SC1810C_CTL_MUTE_SW 1 +#define SC1810C_CTL_AB_SW 3 +#define SC1810C_CTL_48V_SW 4 + +#define SC1810C_SET_STATE_REQ 161 +#define SC1810C_SET_STATE_REQTYPE SC1810C_CMD_REQTYPE +#define SC1810C_SET_STATE_F1 0x64656D73 +#define SC1810C_SET_STATE_F2 0xF4 + +#define SC1810C_GET_STATE_REQ 162 +#define SC1810C_GET_STATE_REQTYPE \ + (USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN) +#define SC1810C_GET_STATE_F1 SC1810C_SET_STATE_F1 +#define SC1810C_GET_STATE_F2 SC1810C_SET_STATE_F2 + +#define SC1810C_STATE_F1_IDX 2 +#define SC1810C_STATE_F2_IDX 3 + +/* + * This packet includes mixer volumes and + * various other fields, it's an extended + * version of ctl_packet, with a and b + * being zero and different f1/f2. + */ +struct s1810c_state_packet { + u32 fields[63]; +}; + +#define SC1810C_STATE_48V_SW 58 +#define SC1810C_STATE_LINE_SW 59 +#define SC1810C_STATE_MUTE_SW 60 +#define SC1810C_STATE_AB_SW 62 + +struct s1810_mixer_state { + uint16_t seqnum; + struct mutex usb_mutex; + struct mutex data_mutex; +}; + +static int +snd_s1810c_send_ctl_packet(struct usb_device *dev, u32 a, + u32 b, u32 c, u32 d, u32 e) +{ + struct s1810c_ctl_packet pkt = { 0 }; + int ret = 0; + + pkt.fixed1 = SC1810C_CMD_F1; + pkt.fixed2 = SC1810C_CMD_F2; + + pkt.a = a; + pkt.b = b; + pkt.c = c; + pkt.d = d; + /* + * Value for settings 0/1 for this + * output channel is always 0 (probably because + * there is no ADAT output on 1810c) + */ + pkt.e = (c == 4) ? 0 : e; + + ret = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + SC1810C_CMD_REQ, + SC1810C_CMD_REQTYPE, 0, 0, &pkt, sizeof(pkt)); + if (ret < 0) { + dev_warn(&dev->dev, "could not send ctl packet\n"); + return ret; + } + return 0; +} + +/* + * When opening Universal Control the program periodicaly + * sends and receives state packets for syncinc state between + * the device and the host. + * + * Note that if we send only the request to get data back we'll + * get an error, we need to first send an empty state packet and + * then ask to receive a filled. Their seqnumbers must also match. + */ +static int +snd_sc1810c_get_status_field(struct usb_device *dev, + u32 *field, int field_idx, uint16_t *seqnum) +{ + struct s1810c_state_packet pkt_out = { { 0 } }; + struct s1810c_state_packet pkt_in = { { 0 } }; + int ret = 0; + + pkt_out.fields[SC1810C_STATE_F1_IDX] = SC1810C_SET_STATE_F1; + pkt_out.fields[SC1810C_STATE_F2_IDX] = SC1810C_SET_STATE_F2; + ret = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), + SC1810C_SET_STATE_REQ, + SC1810C_SET_STATE_REQTYPE, + (*seqnum), 0, &pkt_out, sizeof(pkt_out)); + if (ret < 0) { + dev_warn(&dev->dev, "could not send state packet (%d)\n", ret); + return ret; + } + + ret = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), + SC1810C_GET_STATE_REQ, + SC1810C_GET_STATE_REQTYPE, + (*seqnum), 0, &pkt_in, sizeof(pkt_in)); + if (ret < 0) { + dev_warn(&dev->dev, "could not get state field %u (%d)\n", + field_idx, ret); + return ret; + } + + (*field) = pkt_in.fields[field_idx]; + (*seqnum)++; + return 0; +} + +/* + * This is what I got when bypassing the mixer with + * all channels split. I'm not 100% sure of what's going + * on, I could probably clean this up based on my observations + * but I prefer to keep the same behavior as the windows driver. + */ +static int snd_s1810c_init_mixer_maps(struct snd_usb_audio *chip) +{ + u32 a, b, c, e, n, off; + struct usb_device *dev = chip->dev; + + /* Set initial volume levels ? */ + a = 0x64; + e = 0xbc; + for (n = 0; n < 2; n++) { + off = n * 18; + for (b = off, c = 0; b < 18 + off; b++) { + /* This channel to all outputs ? */ + for (c = 0; c <= 8; c++) { + snd_s1810c_send_ctl_packet(dev, a, b, c, 0, e); + snd_s1810c_send_ctl_packet(dev, a, b, c, 1, e); + } + /* This channel to main output (again) */ + snd_s1810c_send_ctl_packet(dev, a, b, 0, 0, e); + snd_s1810c_send_ctl_packet(dev, a, b, 0, 1, e); + } + /* + * I noticed on UC that DAW channels have different + * initial volumes, so this makes sense. + */ + e = 0xb53bf0; + } + + /* Connect analog outputs ? */ + a = 0x65; + e = 0x01000000; + for (b = 1; b < 3; b++) { + snd_s1810c_send_ctl_packet(dev, a, b, 0, 0, e); + snd_s1810c_send_ctl_packet(dev, a, b, 0, 1, e); + } + snd_s1810c_send_ctl_packet(dev, a, 0, 0, 0, e); + snd_s1810c_send_ctl_packet(dev, a, 0, 0, 1, e); + + /* Set initial volume levels for S/PDIF mappings ? */ + a = 0x64; + e = 0xbc; + c = 3; + for (n = 0; n < 2; n++) { + off = n * 18; + for (b = off; b < 18 + off; b++) { + snd_s1810c_send_ctl_packet(dev, a, b, c, 0, e); + snd_s1810c_send_ctl_packet(dev, a, b, c, 1, e); + } + e = 0xb53bf0; + } + + /* Connect S/PDIF output ? */ + a = 0x65; + e = 0x01000000; + snd_s1810c_send_ctl_packet(dev, a, 3, 0, 0, e); + snd_s1810c_send_ctl_packet(dev, a, 3, 0, 1, e); + + /* Connect all outputs (again) ? */ + a = 0x65; + e = 0x01000000; + for (b = 0; b < 4; b++) { + snd_s1810c_send_ctl_packet(dev, a, b, 0, 0, e); + snd_s1810c_send_ctl_packet(dev, a, b, 0, 1, e); + } + + /* Basic routing to get sound out of the device */ + a = 0x64; + e = 0x01000000; + for (c = 0; c < 4; c++) { + for (b = 0; b < 36; b++) { + if ((c == 0 && b == 18) || /* DAW1/2 -> Main */ + (c == 1 && b == 20) || /* DAW3/4 -> Line3/4 */ + (c == 2 && b == 22) || /* DAW4/5 -> Line5/6 */ + (c == 3 && b == 24)) { /* DAW5/6 -> S/PDIF */ + /* Left */ + snd_s1810c_send_ctl_packet(dev, a, b, c, 0, e); + snd_s1810c_send_ctl_packet(dev, a, b, c, 1, 0); + b++; + /* Right */ + snd_s1810c_send_ctl_packet(dev, a, b, c, 0, 0); + snd_s1810c_send_ctl_packet(dev, a, b, c, 1, e); + } else { + /* Leave the rest disconnected */ + snd_s1810c_send_ctl_packet(dev, a, b, c, 0, 0); + snd_s1810c_send_ctl_packet(dev, a, b, c, 1, 0); + } + } + } + + /* Set initial volume levels for S/PDIF (again) ? */ + a = 0x64; + e = 0xbc; + c = 3; + for (n = 0; n < 2; n++) { + off = n * 18; + for (b = off; b < 18 + off; b++) { + snd_s1810c_send_ctl_packet(dev, a, b, c, 0, e); + snd_s1810c_send_ctl_packet(dev, a, b, c, 1, e); + } + e = 0xb53bf0; + } + + /* Connect S/PDIF outputs (again) ? */ + a = 0x65; + e = 0x01000000; + snd_s1810c_send_ctl_packet(dev, a, 3, 0, 0, e); + snd_s1810c_send_ctl_packet(dev, a, 3, 0, 1, e); + + /* Again ? */ + snd_s1810c_send_ctl_packet(dev, a, 3, 0, 0, e); + snd_s1810c_send_ctl_packet(dev, a, 3, 0, 1, e); + + return 0; +} + +/* + * Sync state with the device and retrieve the requested field, + * whose index is specified in (kctl->private_value & 0xFF), + * from the received fields array. + */ +static int +snd_s1810c_get_switch_state(struct usb_mixer_interface *mixer, + struct snd_kcontrol *kctl, u32 *state) +{ + struct snd_usb_audio *chip = mixer->chip; + struct s1810_mixer_state *private = mixer->private_data; + u32 field = 0; + u32 ctl_idx = (u32) (kctl->private_value & 0xFF); + int ret = 0; + + mutex_lock(&private->usb_mutex); + ret = snd_sc1810c_get_status_field(chip->dev, &field, + ctl_idx, &private->seqnum); + if (ret < 0) + goto unlock; + + *state = field; + unlock: + mutex_unlock(&private->usb_mutex); + return ret ? ret : 0; +} + +/* + * Send a control packet to the device for the control id + * specified in (kctl->private_value >> 8) with value + * specified in (kctl->private_value >> 16). + */ +static int +snd_s1810c_set_switch_state(struct usb_mixer_interface *mixer, + struct snd_kcontrol *kctl) +{ + struct snd_usb_audio *chip = mixer->chip; + struct s1810_mixer_state *private = mixer->private_data; + u32 pval = (u32) kctl->private_value; + u32 ctl_id = (pval >> 8) & 0xFF; + u32 ctl_val = (pval >> 16) & 0x1; + int ret = 0; + + mutex_lock(&private->usb_mutex); + ret = snd_s1810c_send_ctl_packet(chip->dev, 0, 0, 0, ctl_id, ctl_val); + mutex_unlock(&private->usb_mutex); + return ret; +} + +/* Generic get/set/init functions for switch controls */ + +static int +snd_s1810c_switch_get(struct snd_kcontrol *kctl, + struct snd_ctl_elem_value *ctl_elem) +{ + struct usb_mixer_elem_list *list = snd_kcontrol_chip(kctl); + struct usb_mixer_interface *mixer = list->mixer; + struct s1810_mixer_state *private = mixer->private_data; + u32 pval = (u32) kctl->private_value; + u32 ctl_idx = pval & 0xFF; + u32 state = 0; + int ret = 0; + + mutex_lock(&private->data_mutex); + ret = snd_s1810c_get_switch_state(mixer, kctl, &state); + if (ret < 0) + goto unlock; + + switch (ctl_idx) { + case SC1810C_STATE_LINE_SW: + case SC1810C_STATE_AB_SW: + ctl_elem->value.enumerated.item[0] = (int)state; + break; + default: + ctl_elem->value.integer.value[0] = (long)state; + } + + unlock: + mutex_unlock(&private->data_mutex); + return (ret < 0) ? ret : 0; +} + +static int +snd_s1810c_switch_set(struct snd_kcontrol *kctl, + struct snd_ctl_elem_value *ctl_elem) +{ + struct usb_mixer_elem_list *list = snd_kcontrol_chip(kctl); + struct usb_mixer_interface *mixer = list->mixer; + struct s1810_mixer_state *private = mixer->private_data; + u32 pval = (u32) kctl->private_value; + u32 ctl_idx = pval & 0xFF; + u32 curval = 0; + u32 newval = 0; + int ret = 0; + + mutex_lock(&private->data_mutex); + ret = snd_s1810c_get_switch_state(mixer, kctl, &curval); + if (ret < 0) + goto unlock; + + switch (ctl_idx) { + case SC1810C_STATE_LINE_SW: + case SC1810C_STATE_AB_SW: + newval = (u32) ctl_elem->value.enumerated.item[0]; + break; + default: + newval = (u32) ctl_elem->value.integer.value[0]; + } + + if (curval == newval) + goto unlock; + + kctl->private_value &= ~(0x1 << 16); + kctl->private_value |= (unsigned int)(newval & 0x1) << 16; + ret = snd_s1810c_set_switch_state(mixer, kctl); + + unlock: + mutex_unlock(&private->data_mutex); + return (ret < 0) ? 0 : 1; +} + +static int +snd_s1810c_switch_init(struct usb_mixer_interface *mixer, + const struct snd_kcontrol_new *new_kctl) +{ + struct snd_kcontrol *kctl; + struct usb_mixer_elem_info *elem; + + elem = kzalloc(sizeof(struct usb_mixer_elem_info), GFP_KERNEL); + if (!elem) + return -ENOMEM; + + elem->head.mixer = mixer; + elem->control = 0; + elem->head.id = 0; + elem->channels = 1; + + kctl = snd_ctl_new1(new_kctl, elem); + if (!kctl) { + kfree(elem); + return -ENOMEM; + } + kctl->private_free = snd_usb_mixer_elem_free; + + return snd_usb_mixer_add_control(&elem->head, kctl); +} + +static int +snd_s1810c_line_sw_info(struct snd_kcontrol *kctl, + struct snd_ctl_elem_info *uinfo) +{ + static const char *const texts[2] = { + "Preamp On (Mic/Inst)", + "Preamp Off (Line in)" + }; + + return snd_ctl_enum_info(uinfo, 1, ARRAY_SIZE(texts), texts); +} + +static const struct snd_kcontrol_new snd_s1810c_line_sw = { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .name = "Line 1/2 Source Type", + .info = snd_s1810c_line_sw_info, + .get = snd_s1810c_switch_get, + .put = snd_s1810c_switch_set, + .private_value = (SC1810C_STATE_LINE_SW | SC1810C_CTL_LINE_SW << 8) +}; + +static const struct snd_kcontrol_new snd_s1810c_mute_sw = { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .name = "Mute Main Out Switch", + .info = snd_ctl_boolean_mono_info, + .get = snd_s1810c_switch_get, + .put = snd_s1810c_switch_set, + .private_value = (SC1810C_STATE_MUTE_SW | SC1810C_CTL_MUTE_SW << 8) +}; + +static const struct snd_kcontrol_new snd_s1810c_48v_sw = { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .name = "48V Phantom Power On Mic Inputs Switch", + .info = snd_ctl_boolean_mono_info, + .get = snd_s1810c_switch_get, + .put = snd_s1810c_switch_set, + .private_value = (SC1810C_STATE_48V_SW | SC1810C_CTL_48V_SW << 8) +}; + +static int +snd_s1810c_ab_sw_info(struct snd_kcontrol *kctl, + struct snd_ctl_elem_info *uinfo) +{ + static const char *const texts[2] = { + "1/2", + "3/4" + }; + + return snd_ctl_enum_info(uinfo, 1, ARRAY_SIZE(texts), texts); +} + +static const struct snd_kcontrol_new snd_s1810c_ab_sw = { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .name = "Headphone 1 Source Route", + .info = snd_s1810c_ab_sw_info, + .get = snd_s1810c_switch_get, + .put = snd_s1810c_switch_set, + .private_value = (SC1810C_STATE_AB_SW | SC1810C_CTL_AB_SW << 8) +}; + +static void snd_sc1810_mixer_state_free(struct usb_mixer_interface *mixer) +{ + struct s1810_mixer_state *private = mixer->private_data; + kfree(private); + mixer->private_data = NULL; +} + +/* Entry point, called from mixer_quirks.c */ +int snd_sc1810_init_mixer(struct usb_mixer_interface *mixer) +{ + struct s1810_mixer_state *private = NULL; + struct snd_usb_audio *chip = mixer->chip; + struct usb_device *dev = chip->dev; + int ret = 0; + + /* Run this only once */ + if (!list_empty(&chip->mixer_list)) + return 0; + + dev_info(&dev->dev, + "Presonus Studio 1810c, device_setup: %u\n", chip->setup); + if (chip->setup == 1) + dev_info(&dev->dev, "(8out/18in @ 48KHz)\n"); + else if (chip->setup == 2) + dev_info(&dev->dev, "(6out/8in @ 192KHz)\n"); + else + dev_info(&dev->dev, "(8out/14in @ 96KHz)\n"); + + ret = snd_s1810c_init_mixer_maps(chip); + if (ret < 0) + return ret; + + private = kzalloc(sizeof(struct s1810_mixer_state), GFP_KERNEL); + if (!private) + return -ENOMEM; + + mutex_init(&private->usb_mutex); + mutex_init(&private->data_mutex); + + mixer->private_data = private; + mixer->private_free = snd_sc1810_mixer_state_free; + + private->seqnum = 1; + + ret = snd_s1810c_switch_init(mixer, &snd_s1810c_line_sw); + if (ret < 0) + return ret; + + ret = snd_s1810c_switch_init(mixer, &snd_s1810c_mute_sw); + if (ret < 0) + return ret; + + ret = snd_s1810c_switch_init(mixer, &snd_s1810c_48v_sw); + if (ret < 0) + return ret; + + ret = snd_s1810c_switch_init(mixer, &snd_s1810c_ab_sw); + if (ret < 0) + return ret; + return ret; +} diff --git a/sound/usb/mixer_s1810c.h b/sound/usb/mixer_s1810c.h new file mode 100644 index 000000000000..a79a3743cff3 --- /dev/null +++ b/sound/usb/mixer_s1810c.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Presonus Studio 1810c driver for ALSA + * Copyright (C) 2019 Nick Kossifidis + */ + +int snd_sc1810_init_mixer(struct usb_mixer_interface *mixer); diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c index cf5a582df059..2c82990eaacd 100644 --- a/sound/usb/pcm.c +++ b/sound/usb/pcm.c @@ -420,7 +420,12 @@ static int set_sync_ep_implicit_fb_quirk(struct snd_usb_substream *subs, ep = 0x81; ifnum = 1; goto add_sync_ep_from_ifnum; - case USB_ID(0x07fd, 0x0004): /* MOTU MicroBook II */ + case USB_ID(0x07fd, 0x0004): /* MOTU MicroBook II/IIc */ + /* MicroBook IIc */ + if (altsd->bInterfaceClass == USB_CLASS_AUDIO) + return 0; + + /* MicroBook II */ ep = 0x84; ifnum = 0; goto add_sync_ep_from_ifnum; diff --git a/sound/usb/proc.c b/sound/usb/proc.c index ffbf4bd9208c..4174ad11fca6 100644 --- a/sound/usb/proc.c +++ b/sound/usb/proc.c @@ -70,7 +70,7 @@ static void proc_dump_substream_formats(struct snd_usb_substream *subs, struct s snd_iprintf(buffer, " Interface %d\n", fp->iface); snd_iprintf(buffer, " Altset %d\n", fp->altsetting); snd_iprintf(buffer, " Format:"); - for (fmt = 0; fmt <= SNDRV_PCM_FORMAT_LAST; ++fmt) + pcm_for_each_format(fmt) if (fp->formats & pcm_format_to_bits(fmt)) snd_iprintf(buffer, " %s", snd_pcm_format_name(fmt)); diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index d187aa6d50db..1c8719292eee 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -3472,7 +3472,7 @@ AU0828_DEVICE(0x2040, 0x7270, "Hauppauge", "HVR-950Q"), }, /* MOTU Microbook II */ { - USB_DEVICE(0x07fd, 0x0004), + USB_DEVICE_VENDOR_SPEC(0x07fd, 0x0004), .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) { .vendor_name = "MOTU", .product_name = "MicroBookII", diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 7f558f4b4520..86f192a3043d 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -1252,6 +1252,38 @@ static int fasttrackpro_skip_setting_quirk(struct snd_usb_audio *chip, return 0; /* keep this altsetting */ } +static int s1810c_skip_setting_quirk(struct snd_usb_audio *chip, + int iface, int altno) +{ + /* + * Altno settings: + * + * Playback (Interface 1): + * 1: 6 Analog + 2 S/PDIF + * 2: 6 Analog + 2 S/PDIF + * 3: 6 Analog + * + * Capture (Interface 2): + * 1: 8 Analog + 2 S/PDIF + 8 ADAT + * 2: 8 Analog + 2 S/PDIF + 4 ADAT + * 3: 8 Analog + */ + + /* + * I'll leave 2 as the default one and + * use device_setup to switch to the + * other two. + */ + if ((chip->setup == 0 || chip->setup > 2) && altno != 2) + return 1; + else if (chip->setup == 1 && altno != 1) + return 1; + else if (chip->setup == 2 && altno != 3) + return 1; + + return 0; +} + int snd_usb_apply_interface_quirk(struct snd_usb_audio *chip, int iface, int altno) @@ -1265,6 +1297,10 @@ int snd_usb_apply_interface_quirk(struct snd_usb_audio *chip, /* fasttrackpro usb: skip altsets incompatible with device_setup */ if (chip->usb_id == USB_ID(0x0763, 0x2012)) return fasttrackpro_skip_setting_quirk(chip, iface, altno); + /* presonus studio 1810c: skip altsets incompatible with device_setup */ + if (chip->usb_id == USB_ID(0x0194f, 0x010c)) + return s1810c_skip_setting_quirk(chip, iface, altno); + return 0; } @@ -1316,7 +1352,15 @@ int snd_usb_apply_boot_quirk(struct usb_device *dev, case USB_ID(0x2466, 0x8010): /* Fractal Audio Axe-Fx 3 */ return snd_usb_axefx3_boot_quirk(dev); case USB_ID(0x07fd, 0x0004): /* MOTU MicroBook II */ - return snd_usb_motu_microbookii_boot_quirk(dev); + /* + * For some reason interface 3 with vendor-spec class is + * detected on MicroBook IIc. + */ + if (get_iface_desc(intf->altsetting)->bInterfaceClass == + USB_CLASS_VENDOR_SPEC && + get_iface_desc(intf->altsetting)->bInterfaceNumber < 3) + return snd_usb_motu_microbookii_boot_quirk(dev); + break; } return 0; @@ -1754,5 +1798,47 @@ void snd_usb_audioformat_attributes_quirk(struct snd_usb_audio *chip, else fp->ep_attr |= USB_ENDPOINT_SYNC_SYNC; break; + case USB_ID(0x07fd, 0x0004): /* MOTU MicroBook IIc */ + /* + * MaxPacketsOnly attribute is erroneously set in endpoint + * descriptors. As a result this card produces noise with + * all sample rates other than 96 KHz. + */ + fp->attributes &= ~UAC_EP_CS_ATTR_FILL_MAX; + break; } } + +/* + * registration quirk: + * the registration is skipped if a device matches with the given ID, + * unless the interface reaches to the defined one. This is for delaying + * the registration until the last known interface, so that the card and + * devices appear at the same time. + */ + +struct registration_quirk { + unsigned int usb_id; /* composed via USB_ID() */ + unsigned int interface; /* the interface to trigger register */ +}; + +#define REG_QUIRK_ENTRY(vendor, product, iface) \ + { .usb_id = USB_ID(vendor, product), .interface = (iface) } + +static const struct registration_quirk registration_quirks[] = { + REG_QUIRK_ENTRY(0x0951, 0x16d8, 2), /* Kingston HyperX AMP */ + { 0 } /* terminator */ +}; + +/* return true if skipping registration */ +bool snd_usb_registration_quirk(struct snd_usb_audio *chip, int iface) +{ + const struct registration_quirk *q; + + for (q = registration_quirks; q->usb_id; q++) + if (chip->usb_id == q->usb_id) + return iface != q->interface; + + /* Register as normal */ + return false; +} diff --git a/sound/usb/quirks.h b/sound/usb/quirks.h index df0355843a4c..c76cf24a640a 100644 --- a/sound/usb/quirks.h +++ b/sound/usb/quirks.h @@ -51,4 +51,6 @@ void snd_usb_audioformat_attributes_quirk(struct snd_usb_audio *chip, struct audioformat *fp, int stream); +bool snd_usb_registration_quirk(struct snd_usb_audio *chip, int iface); + #endif /* __USBAUDIO_QUIRKS_H */ diff --git a/sound/usb/stream.c b/sound/usb/stream.c index fc5879e7ac5f..d6f0fd967c30 100644 --- a/sound/usb/stream.c +++ b/sound/usb/stream.c @@ -506,6 +506,9 @@ static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip, subs = &as->substream[stream]; if (subs->ep_num) continue; + if (snd_device_get_state(chip->card, as->pcm) != + SNDRV_DEV_BUILD) + chip->need_delayed_register = true; err = snd_pcm_new_stream(as->pcm, stream, 1); if (err < 0) return err; diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index 2d4970edc8ff..7ea1a1dbcd22 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -34,6 +34,7 @@ struct snd_usb_audio { unsigned int txfr_quirk:1; /* Subframe boundaries on transfers */ unsigned int tx_length_quirk:1; /* Put length specifier in transfers */ unsigned int setup_fmt_after_resume_quirk:1; /* setup the format to interface after resume */ + unsigned int need_delayed_register:1; /* warn for delayed registration */ int num_interfaces; int num_suspended_intf; int sample_rate_read_error; diff --git a/sound/usb/usx2y/usbusx2yaudio.c b/sound/usb/usx2y/usbusx2yaudio.c index 772f6f3ccbb1..37d290fe9d43 100644 --- a/sound/usb/usx2y/usbusx2yaudio.c +++ b/sound/usb/usx2y/usbusx2yaudio.c @@ -906,11 +906,12 @@ static const struct snd_pcm_ops snd_usX2Y_pcm_ops = */ static void usX2Y_audio_stream_free(struct snd_usX2Y_substream **usX2Y_substream) { - kfree(usX2Y_substream[SNDRV_PCM_STREAM_PLAYBACK]); - usX2Y_substream[SNDRV_PCM_STREAM_PLAYBACK] = NULL; + int stream; - kfree(usX2Y_substream[SNDRV_PCM_STREAM_CAPTURE]); - usX2Y_substream[SNDRV_PCM_STREAM_CAPTURE] = NULL; + for_each_pcm_streams(stream) { + kfree(usX2Y_substream[stream]); + usX2Y_substream[stream] = NULL; + } } static void snd_usX2Y_pcm_private_free(struct snd_pcm *pcm) diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c index 8cb504d30384..5ef1c15e88ad 100644 --- a/tools/accounting/getdelays.c +++ b/tools/accounting/getdelays.c @@ -136,7 +136,7 @@ static int send_cmd(int sd, __u16 nlmsg_type, __u32 nlmsg_pid, msg.g.version = 0x1; na = (struct nlattr *) GENLMSG_DATA(&msg); na->nla_type = nla_type; - na->nla_len = nla_len + 1 + NLA_HDRLEN; + na->nla_len = nla_len + NLA_HDRLEN; memcpy(NLA_DATA(na), nla_data, nla_len); msg.n.nlmsg_len += NLMSG_ALIGN(na->nla_len); diff --git a/tools/arch/x86/include/asm/unistd_64.h b/tools/arch/x86/include/asm/unistd_64.h index cb52a3a8b8fc..4205ed4158bf 100644 --- a/tools/arch/x86/include/asm/unistd_64.h +++ b/tools/arch/x86/include/asm/unistd_64.h @@ -1,4 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NR_userfaultfd +#define __NR_userfaultfd 282 +#endif #ifndef __NR_perf_event_open # define __NR_perf_event_open 298 #endif diff --git a/tools/kvm/kvm_stat/kvm_stat b/tools/kvm/kvm_stat/kvm_stat index 4cf93110c259..e83fc8e868f4 100755 --- a/tools/kvm/kvm_stat/kvm_stat +++ b/tools/kvm/kvm_stat/kvm_stat @@ -25,7 +25,7 @@ import sys import locale import os import time -import optparse +import argparse import ctypes import fcntl import resource @@ -33,6 +33,8 @@ import struct import re import subprocess from collections import defaultdict, namedtuple +from functools import reduce +from datetime import datetime VMX_EXIT_REASONS = { 'EXCEPTION_NMI': 0, @@ -873,7 +875,7 @@ class Stats(object): if options.debugfs: providers.append(DebugfsProvider(options.pid, options.fields, - options.dbgfs_include_past)) + options.debugfs_include_past)) if options.tracepoints or not providers: providers.append(TracepointProvider(options.pid, options.fields)) @@ -974,15 +976,17 @@ DELAY_DEFAULT = 3.0 MAX_GUEST_NAME_LEN = 48 MAX_REGEX_LEN = 44 SORT_DEFAULT = 0 +MIN_DELAY = 0.1 +MAX_DELAY = 25.5 class Tui(object): """Instruments curses to draw a nice text ui.""" - def __init__(self, stats): + def __init__(self, stats, opts): self.stats = stats self.screen = None self._delay_initial = 0.25 - self._delay_regular = DELAY_DEFAULT + self._delay_regular = opts.set_delay self._sorting = SORT_DEFAULT self._display_guests = 0 @@ -1183,7 +1187,7 @@ class Tui(object): if not self._is_running_guest(self.stats.pid_filter): if self._gname: - try: # ...to identify the guest by name in case it's back + try: # ...to identify the guest by name in case it's back pids = self.get_pid_from_gname(self._gname) if len(pids) == 1: self._refresh_header(pids[0]) @@ -1282,7 +1286,8 @@ class Tui(object): ' p filter by guest name/PID', ' q quit', ' r reset stats', - ' s set update interval', + ' s set delay between refreshs (value range: ' + '%s-%s secs)' % (MIN_DELAY, MAX_DELAY), ' x toggle reporting of stats for individual child trace' ' events', 'Any other key refreshes statistics immediately') @@ -1336,8 +1341,8 @@ class Tui(object): msg = '' while True: self.screen.erase() - self.screen.addstr(0, 0, 'Set update interval (defaults to %.1fs).' % - DELAY_DEFAULT, curses.A_BOLD) + self.screen.addstr(0, 0, 'Set update interval (defaults to %.1fs).' + % DELAY_DEFAULT, curses.A_BOLD) self.screen.addstr(4, 0, msg) self.screen.addstr(2, 0, 'Change delay from %.1fs to ' % self._delay_regular) @@ -1348,11 +1353,9 @@ class Tui(object): try: if len(val) > 0: delay = float(val) - if delay < 0.1: - msg = '"' + str(val) + '": Value must be >=0.1' - continue - if delay > 25.5: - msg = '"' + str(val) + '": Value must be <=25.5' + err = is_delay_valid(delay) + if err is not None: + msg = err continue else: delay = DELAY_DEFAULT @@ -1488,33 +1491,64 @@ def batch(stats): pass -def log(stats): +class StdFormat(object): + def __init__(self, keys): + self._banner = '' + for key in keys: + self._banner += key.split(' ')[0] + ' ' + + def get_banner(self): + return self._banner + + @staticmethod + def get_statline(keys, s): + res = '' + for key in keys: + res += ' %9d' % s[key].delta + return res + + +class CSVFormat(object): + def __init__(self, keys): + self._banner = 'timestamp' + self._banner += reduce(lambda res, key: "{},{!s}".format(res, + key.split(' ')[0]), keys, '') + + def get_banner(self): + return self._banner + + @staticmethod + def get_statline(keys, s): + return reduce(lambda res, key: "{},{!s}".format(res, s[key].delta), + keys, '') + + +def log(stats, opts, frmt, keys): """Prints statistics as reiterating key block, multiple value blocks.""" - keys = sorted(stats.get().keys()) - - def banner(): - for key in keys: - print(key.split(' ')[0], end=' ') - print() - - def statline(): - s = stats.get() - for key in keys: - print(' %9d' % s[key].delta, end=' ') - print() line = 0 banner_repeat = 20 while True: try: - time.sleep(1) + time.sleep(opts.set_delay) if line % banner_repeat == 0: - banner() - statline() + print(frmt.get_banner()) + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + frmt.get_statline(keys, stats.get())) line += 1 except KeyboardInterrupt: break +def is_delay_valid(delay): + """Verify delay is in valid value range.""" + msg = None + if delay < MIN_DELAY: + msg = '"' + str(delay) + '": Delay must be >=%s' % MIN_DELAY + if delay > MAX_DELAY: + msg = '"' + str(delay) + '": Delay must be <=%s' % MAX_DELAY + return msg + + def get_options(): """Returns processed program arguments.""" description_text = """ @@ -1545,89 +1579,85 @@ Interactive Commands: p filter by PID q quit r reset stats - s set update interval + s set update interval (value range: 0.1-25.5 secs) x toggle reporting of stats for individual child trace events Press any other key to refresh statistics immediately. """ % (PATH_DEBUGFS_KVM, PATH_DEBUGFS_TRACING) - class PlainHelpFormatter(optparse.IndentedHelpFormatter): - def format_description(self, description): - if description: - return description + "\n" - else: - return "" + class Guest_to_pid(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + try: + pids = Tui.get_pid_from_gname(values) + except: + sys.exit('Error while searching for guest "{}". Use "-p" to ' + 'specify a pid instead?'.format(values)) + if len(pids) == 0: + sys.exit('Error: No guest by the name "{}" found' + .format(values)) + if len(pids) > 1: + sys.exit('Error: Multiple processes found (pids: {}). Use "-p"' + ' to specify the desired pid'.format(" ".join(pids))) + namespace.pid = pids[0] - def cb_guest_to_pid(option, opt, val, parser): - try: - pids = Tui.get_pid_from_gname(val) - except: - sys.exit('Error while searching for guest "{}". Use "-p" to ' - 'specify a pid instead?'.format(val)) - if len(pids) == 0: - sys.exit('Error: No guest by the name "{}" found'.format(val)) - if len(pids) > 1: - sys.exit('Error: Multiple processes found (pids: {}). Use "-p" ' - 'to specify the desired pid'.format(" ".join(pids))) - parser.values.pid = pids[0] - - optparser = optparse.OptionParser(description=description_text, - formatter=PlainHelpFormatter()) - optparser.add_option('-1', '--once', '--batch', - action='store_true', - default=False, - dest='once', - help='run in batch mode for one second', - ) - optparser.add_option('-i', '--debugfs-include-past', - action='store_true', - default=False, - dest='dbgfs_include_past', - help='include all available data on past events for ' - 'debugfs', - ) - optparser.add_option('-l', '--log', - action='store_true', - default=False, - dest='log', - help='run in logging mode (like vmstat)', - ) - optparser.add_option('-t', '--tracepoints', - action='store_true', - default=False, - dest='tracepoints', - help='retrieve statistics from tracepoints', - ) - optparser.add_option('-d', '--debugfs', - action='store_true', - default=False, - dest='debugfs', - help='retrieve statistics from debugfs', - ) - optparser.add_option('-f', '--fields', - action='store', - default='', - dest='fields', - help='''fields to display (regex) - "-f help" for a list of available events''', - ) - optparser.add_option('-p', '--pid', - action='store', - default=0, - type='int', - dest='pid', - help='restrict statistics to pid', - ) - optparser.add_option('-g', '--guest', - action='callback', - type='string', - dest='pid', - metavar='GUEST', - help='restrict statistics to guest by name', - callback=cb_guest_to_pid, - ) - options, unkn = optparser.parse_args(sys.argv) - if len(unkn) != 1: - sys.exit('Error: Extra argument(s): ' + ' '.join(unkn[1:])) + argparser = argparse.ArgumentParser(description=description_text, + formatter_class=argparse + .RawTextHelpFormatter) + argparser.add_argument('-1', '--once', '--batch', + action='store_true', + default=False, + help='run in batch mode for one second', + ) + argparser.add_argument('-c', '--csv', + action='store_true', + default=False, + help='log in csv format - requires option -l/--log', + ) + argparser.add_argument('-d', '--debugfs', + action='store_true', + default=False, + help='retrieve statistics from debugfs', + ) + argparser.add_argument('-f', '--fields', + default='', + help='''fields to display (regex) +"-f help" for a list of available events''', + ) + argparser.add_argument('-g', '--guest', + type=str, + help='restrict statistics to guest by name', + action=Guest_to_pid, + ) + argparser.add_argument('-i', '--debugfs-include-past', + action='store_true', + default=False, + help='include all available data on past events for' + ' debugfs', + ) + argparser.add_argument('-l', '--log', + action='store_true', + default=False, + help='run in logging mode (like vmstat)', + ) + argparser.add_argument('-p', '--pid', + type=int, + default=0, + help='restrict statistics to pid', + ) + argparser.add_argument('-s', '--set-delay', + type=float, + default=DELAY_DEFAULT, + metavar='DELAY', + help='set delay between refreshs (value range: ' + '%s-%s secs)' % (MIN_DELAY, MAX_DELAY), + ) + argparser.add_argument('-t', '--tracepoints', + action='store_true', + default=False, + help='retrieve statistics from tracepoints', + ) + options = argparser.parse_args() + if options.csv and not options.log: + sys.exit('Error: Option -c/--csv requires -l/--log') try: # verify that we were passed a valid regex up front re.compile(options.fields) @@ -1693,6 +1723,10 @@ def main(): sys.stderr.write('Did you use a (unsupported) tid instead of a pid?\n') sys.exit('Specified pid does not exist.') + err = is_delay_valid(options.set_delay) + if err is not None: + sys.exit('Error: ' + err) + stats = Stats(options) if options.fields == 'help': @@ -1704,12 +1738,18 @@ def main(): sys.exit(0) if options.log: - log(stats) + keys = sorted(stats.get().keys()) + if options.csv: + frmt = CSVFormat(keys) + else: + frmt = StdFormat(keys) + log(stats, options, frmt, keys) elif not options.once: - with Tui(stats) as tui: + with Tui(stats, options) as tui: tui.show_stats() else: batch(stats) + if __name__ == "__main__": main() diff --git a/tools/kvm/kvm_stat/kvm_stat.txt b/tools/kvm/kvm_stat/kvm_stat.txt index c057ba52364e..a97ded2aedad 100644 --- a/tools/kvm/kvm_stat/kvm_stat.txt +++ b/tools/kvm/kvm_stat/kvm_stat.txt @@ -49,7 +49,7 @@ INTERACTIVE COMMANDS *r*:: reset stats -*s*:: set update interval +*s*:: set delay between refreshs *x*:: toggle reporting of stats for child trace events :: *Note*: The stats for the parents summarize the respective child trace @@ -64,37 +64,45 @@ OPTIONS --batch:: run in batch mode for one second --l:: ---log:: - run in logging mode (like vmstat) - --t:: ---tracepoints:: - retrieve statistics from tracepoints +-c:: +--csv=:: + log in csv format - requires option -l/--log -d:: --debugfs:: retrieve statistics from debugfs +-f:: +--fields=:: + fields to display (regex), "-f help" for a list of available events + +-g:: +--guest=:: + limit statistics to one virtual machine (guest name) + +-h:: +--help:: + show help message + -i:: --debugfs-include-past:: include all available data on past events for debugfs +-l:: +--log:: + run in logging mode (like vmstat) + -p:: --pid=:: limit statistics to one virtual machine (pid) --g:: ---guest=:: - limit statistics to one virtual machine (guest name) +-s:: +--set-delay:: + set delay between refreshs (value range: 0.1-25.5 secs) --f:: ---fields=:: - fields to display (regex), "-f help" for a list of available events - --h:: ---help:: - show help message +-t:: +--tracepoints:: + retrieve statistics from tracepoints SEE ALSO -------- diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 30072c3f52fb..16877c3daabf 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -1,13 +1,16 @@ -/s390x/sync_regs_test /s390x/memop +/s390x/resets +/s390x/sync_regs_test /x86_64/cr4_cpuid_sync_test /x86_64/evmcs_test /x86_64/hyperv_cpuid /x86_64/mmio_warning_test /x86_64/platform_info_test +/x86_64/set_memory_region_test /x86_64/set_sregs_test /x86_64/smm_test /x86_64/state_test +/x86_64/svm_vmcall_test /x86_64/sync_regs_test /x86_64/vmx_close_while_nested_test /x86_64/vmx_dirty_log_test @@ -15,5 +18,7 @@ /x86_64/vmx_tsc_adjust_test /x86_64/xss_msr_test /clear_dirty_log_test +/demand_paging_test /dirty_log_test /kvm_create_max_vcpus +/steal_time diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index d91c53b726e6..712a2ddd2a27 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -7,7 +7,7 @@ top_srcdir = ../../../.. KSFT_KHDR_INSTALL := 1 UNAME_M := $(shell uname -m) -LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/sparsebit.c +LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/sparsebit.c lib/test_util.c LIBKVM_x86_64 = lib/x86_64/processor.c lib/x86_64/vmx.c lib/x86_64/svm.c lib/x86_64/ucall.c LIBKVM_aarch64 = lib/aarch64/processor.c lib/aarch64/ucall.c LIBKVM_s390x = lib/s390x/processor.c lib/s390x/ucall.c @@ -17,27 +17,33 @@ TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid TEST_GEN_PROGS_x86_64 += x86_64/mmio_warning_test TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test +TEST_GEN_PROGS_x86_64 += x86_64/set_memory_region_test TEST_GEN_PROGS_x86_64 += x86_64/set_sregs_test TEST_GEN_PROGS_x86_64 += x86_64/smm_test TEST_GEN_PROGS_x86_64 += x86_64/state_test +TEST_GEN_PROGS_x86_64 += x86_64/svm_vmcall_test TEST_GEN_PROGS_x86_64 += x86_64/sync_regs_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_close_while_nested_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_dirty_log_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_set_nested_state_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_tsc_adjust_test TEST_GEN_PROGS_x86_64 += x86_64/xss_msr_test -TEST_GEN_PROGS_x86_64 += x86_64/svm_vmcall_test TEST_GEN_PROGS_x86_64 += clear_dirty_log_test +TEST_GEN_PROGS_x86_64 += demand_paging_test TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus +TEST_GEN_PROGS_x86_64 += steal_time TEST_GEN_PROGS_aarch64 += clear_dirty_log_test +TEST_GEN_PROGS_aarch64 += demand_paging_test TEST_GEN_PROGS_aarch64 += dirty_log_test TEST_GEN_PROGS_aarch64 += kvm_create_max_vcpus +TEST_GEN_PROGS_aarch64 += steal_time TEST_GEN_PROGS_s390x = s390x/memop -TEST_GEN_PROGS_s390x += s390x/sync_regs_test TEST_GEN_PROGS_s390x += s390x/resets +TEST_GEN_PROGS_s390x += s390x/sync_regs_test +TEST_GEN_PROGS_s390x += demand_paging_test TEST_GEN_PROGS_s390x += dirty_log_test TEST_GEN_PROGS_s390x += kvm_create_max_vcpus diff --git a/tools/testing/selftests/kvm/clear_dirty_log_test.c b/tools/testing/selftests/kvm/clear_dirty_log_test.c index 749336937d37..11672ec6f74e 100644 --- a/tools/testing/selftests/kvm/clear_dirty_log_test.c +++ b/tools/testing/selftests/kvm/clear_dirty_log_test.c @@ -1,2 +1,6 @@ #define USE_CLEAR_DIRTY_LOG +#define KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE (1 << 0) +#define KVM_DIRTY_LOG_INITIALLY_SET (1 << 1) +#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ + KVM_DIRTY_LOG_INITIALLY_SET) #include "dirty_log_test.c" diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c new file mode 100644 index 000000000000..360cd3ea4cd6 --- /dev/null +++ b/tools/testing/selftests/kvm/demand_paging_test.c @@ -0,0 +1,661 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KVM demand paging test + * Adapted from dirty_log_test.c + * + * Copyright (C) 2018, Red Hat, Inc. + * Copyright (C) 2019, Google, Inc. + */ + +#define _GNU_SOURCE /* for program_invocation_name */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" + +#ifdef __NR_userfaultfd + +/* The memory slot index demand page */ +#define TEST_MEM_SLOT_INDEX 1 + +/* Default guest test virtual memory offset */ +#define DEFAULT_GUEST_TEST_MEM 0xc0000000 + +#define DEFAULT_GUEST_TEST_MEM_SIZE (1 << 30) /* 1G */ + +#ifdef PRINT_PER_PAGE_UPDATES +#define PER_PAGE_DEBUG(...) printf(__VA_ARGS__) +#else +#define PER_PAGE_DEBUG(...) _no_printf(__VA_ARGS__) +#endif + +#ifdef PRINT_PER_VCPU_UPDATES +#define PER_VCPU_DEBUG(...) printf(__VA_ARGS__) +#else +#define PER_VCPU_DEBUG(...) _no_printf(__VA_ARGS__) +#endif + +#define MAX_VCPUS 512 + +/* + * Guest/Host shared variables. Ensure addr_gva2hva() and/or + * sync_global_to/from_guest() are used when accessing from + * the host. READ/WRITE_ONCE() should also be used with anything + * that may change. + */ +static uint64_t host_page_size; +static uint64_t guest_page_size; + +static char *guest_data_prototype; + +/* + * Guest physical memory offset of the testing memory slot. + * This will be set to the topmost valid physical address minus + * the test memory size. + */ +static uint64_t guest_test_phys_mem; + +/* + * Guest virtual memory offset of the testing memory slot. + * Must not conflict with identity mapped test code. + */ +static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM; + +struct vcpu_args { + uint64_t gva; + uint64_t pages; + + /* Only used by the host userspace part of the vCPU thread */ + int vcpu_id; + struct kvm_vm *vm; +}; + +static struct vcpu_args vcpu_args[MAX_VCPUS]; + +/* + * Continuously write to the first 8 bytes of each page in the demand paging + * memory region. + */ +static void guest_code(uint32_t vcpu_id) +{ + uint64_t gva; + uint64_t pages; + int i; + + /* Make sure vCPU args data structure is not corrupt. */ + GUEST_ASSERT(vcpu_args[vcpu_id].vcpu_id == vcpu_id); + + gva = vcpu_args[vcpu_id].gva; + pages = vcpu_args[vcpu_id].pages; + + for (i = 0; i < pages; i++) { + uint64_t addr = gva + (i * guest_page_size); + + addr &= ~(host_page_size - 1); + *(uint64_t *)addr = 0x0123456789ABCDEF; + } + + GUEST_SYNC(1); +} + +static void *vcpu_worker(void *data) +{ + int ret; + struct vcpu_args *args = (struct vcpu_args *)data; + struct kvm_vm *vm = args->vm; + int vcpu_id = args->vcpu_id; + struct kvm_run *run; + struct timespec start, end, ts_diff; + + vcpu_args_set(vm, vcpu_id, 1, vcpu_id); + run = vcpu_state(vm, vcpu_id); + + clock_gettime(CLOCK_MONOTONIC, &start); + + /* Let the guest access its memory */ + ret = _vcpu_run(vm, vcpu_id); + TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret); + if (get_ucall(vm, vcpu_id, NULL) != UCALL_SYNC) { + TEST_ASSERT(false, + "Invalid guest sync status: exit_reason=%s\n", + exit_reason_str(run->exit_reason)); + } + + clock_gettime(CLOCK_MONOTONIC, &end); + ts_diff = timespec_sub(end, start); + PER_VCPU_DEBUG("vCPU %d execution time: %ld.%.9lds\n", vcpu_id, + ts_diff.tv_sec, ts_diff.tv_nsec); + + return NULL; +} + +#define PAGE_SHIFT_4K 12 +#define PTES_PER_4K_PT 512 + +static struct kvm_vm *create_vm(enum vm_guest_mode mode, int vcpus, + uint64_t vcpu_memory_bytes) +{ + struct kvm_vm *vm; + uint64_t pages = DEFAULT_GUEST_PHY_PAGES; + + /* Account for a few pages per-vCPU for stacks */ + pages += DEFAULT_STACK_PGS * vcpus; + + /* + * Reserve twice the ammount of memory needed to map the test region and + * the page table / stacks region, at 4k, for page tables. Do the + * calculation with 4K page size: the smallest of all archs. (e.g., 64K + * page size guest will need even less memory for page tables). + */ + pages += (2 * pages) / PTES_PER_4K_PT; + pages += ((2 * vcpus * vcpu_memory_bytes) >> PAGE_SHIFT_4K) / + PTES_PER_4K_PT; + pages = vm_adjust_num_guest_pages(mode, pages); + + pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode)); + + vm = _vm_create(mode, pages, O_RDWR); + kvm_vm_elf_load(vm, program_invocation_name, 0, 0); +#ifdef __x86_64__ + vm_create_irqchip(vm); +#endif + return vm; +} + +static int handle_uffd_page_request(int uffd, uint64_t addr) +{ + pid_t tid; + struct timespec start; + struct timespec end; + struct uffdio_copy copy; + int r; + + tid = syscall(__NR_gettid); + + copy.src = (uint64_t)guest_data_prototype; + copy.dst = addr; + copy.len = host_page_size; + copy.mode = 0; + + clock_gettime(CLOCK_MONOTONIC, &start); + + r = ioctl(uffd, UFFDIO_COPY, ©); + if (r == -1) { + pr_info("Failed Paged in 0x%lx from thread %d with errno: %d\n", + addr, tid, errno); + return r; + } + + clock_gettime(CLOCK_MONOTONIC, &end); + + PER_PAGE_DEBUG("UFFDIO_COPY %d \t%ld ns\n", tid, + timespec_to_ns(timespec_sub(end, start))); + PER_PAGE_DEBUG("Paged in %ld bytes at 0x%lx from thread %d\n", + host_page_size, addr, tid); + + return 0; +} + +bool quit_uffd_thread; + +struct uffd_handler_args { + int uffd; + int pipefd; + useconds_t delay; +}; + +static void *uffd_handler_thread_fn(void *arg) +{ + struct uffd_handler_args *uffd_args = (struct uffd_handler_args *)arg; + int uffd = uffd_args->uffd; + int pipefd = uffd_args->pipefd; + useconds_t delay = uffd_args->delay; + int64_t pages = 0; + struct timespec start, end, ts_diff; + + clock_gettime(CLOCK_MONOTONIC, &start); + while (!quit_uffd_thread) { + struct uffd_msg msg; + struct pollfd pollfd[2]; + char tmp_chr; + int r; + uint64_t addr; + + pollfd[0].fd = uffd; + pollfd[0].events = POLLIN; + pollfd[1].fd = pipefd; + pollfd[1].events = POLLIN; + + r = poll(pollfd, 2, -1); + switch (r) { + case -1: + pr_info("poll err"); + continue; + case 0: + continue; + case 1: + break; + default: + pr_info("Polling uffd returned %d", r); + return NULL; + } + + if (pollfd[0].revents & POLLERR) { + pr_info("uffd revents has POLLERR"); + return NULL; + } + + if (pollfd[1].revents & POLLIN) { + r = read(pollfd[1].fd, &tmp_chr, 1); + TEST_ASSERT(r == 1, + "Error reading pipefd in UFFD thread\n"); + return NULL; + } + + if (!pollfd[0].revents & POLLIN) + continue; + + r = read(uffd, &msg, sizeof(msg)); + if (r == -1) { + if (errno == EAGAIN) + continue; + pr_info("Read of uffd gor errno %d", errno); + return NULL; + } + + if (r != sizeof(msg)) { + pr_info("Read on uffd returned unexpected size: %d bytes", r); + return NULL; + } + + if (!(msg.event & UFFD_EVENT_PAGEFAULT)) + continue; + + if (delay) + usleep(delay); + addr = msg.arg.pagefault.address; + r = handle_uffd_page_request(uffd, addr); + if (r < 0) + return NULL; + pages++; + } + + clock_gettime(CLOCK_MONOTONIC, &end); + ts_diff = timespec_sub(end, start); + PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n", + pages, ts_diff.tv_sec, ts_diff.tv_nsec, + pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0)); + + return NULL; +} + +static int setup_demand_paging(struct kvm_vm *vm, + pthread_t *uffd_handler_thread, int pipefd, + useconds_t uffd_delay, + struct uffd_handler_args *uffd_args, + void *hva, uint64_t len) +{ + int uffd; + struct uffdio_api uffdio_api; + struct uffdio_register uffdio_register; + + uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); + if (uffd == -1) { + pr_info("uffd creation failed\n"); + return -1; + } + + uffdio_api.api = UFFD_API; + uffdio_api.features = 0; + if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) { + pr_info("ioctl uffdio_api failed\n"); + return -1; + } + + uffdio_register.range.start = (uint64_t)hva; + uffdio_register.range.len = len; + uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) { + pr_info("ioctl uffdio_register failed\n"); + return -1; + } + + if ((uffdio_register.ioctls & UFFD_API_RANGE_IOCTLS) != + UFFD_API_RANGE_IOCTLS) { + pr_info("unexpected userfaultfd ioctl set\n"); + return -1; + } + + uffd_args->uffd = uffd; + uffd_args->pipefd = pipefd; + uffd_args->delay = uffd_delay; + pthread_create(uffd_handler_thread, NULL, uffd_handler_thread_fn, + uffd_args); + + PER_VCPU_DEBUG("Created uffd thread for HVA range [%p, %p)\n", + hva, hva + len); + + return 0; +} + +static void run_test(enum vm_guest_mode mode, bool use_uffd, + useconds_t uffd_delay, int vcpus, + uint64_t vcpu_memory_bytes) +{ + pthread_t *vcpu_threads; + pthread_t *uffd_handler_threads = NULL; + struct uffd_handler_args *uffd_args = NULL; + struct timespec start, end, ts_diff; + int *pipefds = NULL; + struct kvm_vm *vm; + uint64_t guest_num_pages; + int vcpu_id; + int r; + + vm = create_vm(mode, vcpus, vcpu_memory_bytes); + + guest_page_size = vm_get_page_size(vm); + + TEST_ASSERT(vcpu_memory_bytes % guest_page_size == 0, + "Guest memory size is not guest page size aligned."); + + guest_num_pages = (vcpus * vcpu_memory_bytes) / guest_page_size; + guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages); + + /* + * If there should be more memory in the guest test region than there + * can be pages in the guest, it will definitely cause problems. + */ + TEST_ASSERT(guest_num_pages < vm_get_max_gfn(vm), + "Requested more guest memory than address space allows.\n" + " guest pages: %lx max gfn: %x vcpus: %d wss: %lx]\n", + guest_num_pages, vm_get_max_gfn(vm), vcpus, + vcpu_memory_bytes); + + host_page_size = getpagesize(); + TEST_ASSERT(vcpu_memory_bytes % host_page_size == 0, + "Guest memory size is not host page size aligned."); + + guest_test_phys_mem = (vm_get_max_gfn(vm) - guest_num_pages) * + guest_page_size; + guest_test_phys_mem &= ~(host_page_size - 1); + +#ifdef __s390x__ + /* Align to 1M (segment size) */ + guest_test_phys_mem &= ~((1 << 20) - 1); +#endif + + pr_info("guest physical test memory offset: 0x%lx\n", guest_test_phys_mem); + + /* Add an extra memory slot for testing demand paging */ + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, + guest_test_phys_mem, + TEST_MEM_SLOT_INDEX, + guest_num_pages, 0); + + /* Do mapping for the demand paging memory slot */ + virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages, 0); + + ucall_init(vm, NULL); + + guest_data_prototype = malloc(host_page_size); + TEST_ASSERT(guest_data_prototype, + "Failed to allocate buffer for guest data pattern"); + memset(guest_data_prototype, 0xAB, host_page_size); + + vcpu_threads = malloc(vcpus * sizeof(*vcpu_threads)); + TEST_ASSERT(vcpu_threads, "Memory allocation failed"); + + if (use_uffd) { + uffd_handler_threads = + malloc(vcpus * sizeof(*uffd_handler_threads)); + TEST_ASSERT(uffd_handler_threads, "Memory allocation failed"); + + uffd_args = malloc(vcpus * sizeof(*uffd_args)); + TEST_ASSERT(uffd_args, "Memory allocation failed"); + + pipefds = malloc(sizeof(int) * vcpus * 2); + TEST_ASSERT(pipefds, "Unable to allocate memory for pipefd"); + } + + for (vcpu_id = 0; vcpu_id < vcpus; vcpu_id++) { + vm_paddr_t vcpu_gpa; + void *vcpu_hva; + + vm_vcpu_add_default(vm, vcpu_id, guest_code); + + vcpu_gpa = guest_test_phys_mem + (vcpu_id * vcpu_memory_bytes); + PER_VCPU_DEBUG("Added VCPU %d with test mem gpa [%lx, %lx)\n", + vcpu_id, vcpu_gpa, vcpu_gpa + vcpu_memory_bytes); + + /* Cache the HVA pointer of the region */ + vcpu_hva = addr_gpa2hva(vm, vcpu_gpa); + + if (use_uffd) { + /* + * Set up user fault fd to handle demand paging + * requests. + */ + r = pipe2(&pipefds[vcpu_id * 2], + O_CLOEXEC | O_NONBLOCK); + TEST_ASSERT(!r, "Failed to set up pipefd"); + + r = setup_demand_paging(vm, + &uffd_handler_threads[vcpu_id], + pipefds[vcpu_id * 2], + uffd_delay, &uffd_args[vcpu_id], + vcpu_hva, vcpu_memory_bytes); + if (r < 0) + exit(-r); + } + +#ifdef __x86_64__ + vcpu_set_cpuid(vm, vcpu_id, kvm_get_supported_cpuid()); +#endif + + vcpu_args[vcpu_id].vm = vm; + vcpu_args[vcpu_id].vcpu_id = vcpu_id; + vcpu_args[vcpu_id].gva = guest_test_virt_mem + + (vcpu_id * vcpu_memory_bytes); + vcpu_args[vcpu_id].pages = vcpu_memory_bytes / guest_page_size; + } + + /* Export the shared variables to the guest */ + sync_global_to_guest(vm, host_page_size); + sync_global_to_guest(vm, guest_page_size); + sync_global_to_guest(vm, vcpu_args); + + pr_info("Finished creating vCPUs and starting uffd threads\n"); + + clock_gettime(CLOCK_MONOTONIC, &start); + + for (vcpu_id = 0; vcpu_id < vcpus; vcpu_id++) { + pthread_create(&vcpu_threads[vcpu_id], NULL, vcpu_worker, + &vcpu_args[vcpu_id]); + } + + pr_info("Started all vCPUs\n"); + + /* Wait for the vcpu threads to quit */ + for (vcpu_id = 0; vcpu_id < vcpus; vcpu_id++) { + pthread_join(vcpu_threads[vcpu_id], NULL); + PER_VCPU_DEBUG("Joined thread for vCPU %d\n", vcpu_id); + } + + pr_info("All vCPU threads joined\n"); + + clock_gettime(CLOCK_MONOTONIC, &end); + + if (use_uffd) { + char c; + + /* Tell the user fault fd handler threads to quit */ + for (vcpu_id = 0; vcpu_id < vcpus; vcpu_id++) { + r = write(pipefds[vcpu_id * 2 + 1], &c, 1); + TEST_ASSERT(r == 1, "Unable to write to pipefd"); + + pthread_join(uffd_handler_threads[vcpu_id], NULL); + } + } + + ts_diff = timespec_sub(end, start); + pr_info("Total guest execution time: %ld.%.9lds\n", + ts_diff.tv_sec, ts_diff.tv_nsec); + pr_info("Overall demand paging rate: %f pgs/sec\n", + guest_num_pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0)); + + ucall_uninit(vm); + kvm_vm_free(vm); + + free(guest_data_prototype); + free(vcpu_threads); + if (use_uffd) { + free(uffd_handler_threads); + free(uffd_args); + free(pipefds); + } +} + +struct guest_mode { + bool supported; + bool enabled; +}; +static struct guest_mode guest_modes[NUM_VM_MODES]; + +#define guest_mode_init(mode, supported, enabled) ({ \ + guest_modes[mode] = (struct guest_mode){ supported, enabled }; \ +}) + +static void help(char *name) +{ + int i; + + puts(""); + printf("usage: %s [-h] [-m mode] [-u] [-d uffd_delay_usec]\n" + " [-b memory] [-v vcpus]\n", name); + printf(" -m: specify the guest mode ID to test\n" + " (default: test all supported modes)\n" + " This option may be used multiple times.\n" + " Guest mode IDs:\n"); + for (i = 0; i < NUM_VM_MODES; ++i) { + printf(" %d: %s%s\n", i, vm_guest_mode_string(i), + guest_modes[i].supported ? " (supported)" : ""); + } + printf(" -u: use User Fault FD to handle vCPU page\n" + " faults.\n"); + printf(" -d: add a delay in usec to the User Fault\n" + " FD handler to simulate demand paging\n" + " overheads. Ignored without -u.\n"); + printf(" -b: specify the size of the memory region which should be\n" + " demand paged by each vCPU. e.g. 10M or 3G.\n" + " Default: 1G\n"); + printf(" -v: specify the number of vCPUs to run.\n"); + puts(""); + exit(0); +} + +int main(int argc, char *argv[]) +{ + bool mode_selected = false; + uint64_t vcpu_memory_bytes = DEFAULT_GUEST_TEST_MEM_SIZE; + int vcpus = 1; + unsigned int mode; + int opt, i; + bool use_uffd = false; + useconds_t uffd_delay = 0; + +#ifdef __x86_64__ + guest_mode_init(VM_MODE_PXXV48_4K, true, true); +#endif +#ifdef __aarch64__ + guest_mode_init(VM_MODE_P40V48_4K, true, true); + guest_mode_init(VM_MODE_P40V48_64K, true, true); + { + unsigned int limit = kvm_check_cap(KVM_CAP_ARM_VM_IPA_SIZE); + + if (limit >= 52) + guest_mode_init(VM_MODE_P52V48_64K, true, true); + if (limit >= 48) { + guest_mode_init(VM_MODE_P48V48_4K, true, true); + guest_mode_init(VM_MODE_P48V48_64K, true, true); + } + } +#endif +#ifdef __s390x__ + guest_mode_init(VM_MODE_P40V48_4K, true, true); +#endif + + while ((opt = getopt(argc, argv, "hm:ud:b:v:")) != -1) { + switch (opt) { + case 'm': + if (!mode_selected) { + for (i = 0; i < NUM_VM_MODES; ++i) + guest_modes[i].enabled = false; + mode_selected = true; + } + mode = strtoul(optarg, NULL, 10); + TEST_ASSERT(mode < NUM_VM_MODES, + "Guest mode ID %d too big", mode); + guest_modes[mode].enabled = true; + break; + case 'u': + use_uffd = true; + break; + case 'd': + uffd_delay = strtoul(optarg, NULL, 0); + TEST_ASSERT(uffd_delay >= 0, + "A negative UFFD delay is not supported."); + break; + case 'b': + vcpu_memory_bytes = parse_size(optarg); + break; + case 'v': + vcpus = atoi(optarg); + TEST_ASSERT(vcpus > 0, + "Must have a positive number of vCPUs"); + TEST_ASSERT(vcpus <= MAX_VCPUS, + "This test does not currently support\n" + "more than %d vCPUs.", MAX_VCPUS); + break; + case 'h': + default: + help(argv[0]); + break; + } + } + + for (i = 0; i < NUM_VM_MODES; ++i) { + if (!guest_modes[i].enabled) + continue; + TEST_ASSERT(guest_modes[i].supported, + "Guest mode ID %d (%s) not supported.", + i, vm_guest_mode_string(i)); + run_test(i, use_uffd, uffd_delay, vcpus, vcpu_memory_bytes); + } + + return 0; +} + +#else /* __NR_userfaultfd */ + +#warning "missing __NR_userfaultfd definition" + +int main(void) +{ + print_skip("__NR_userfaultfd must be present for userfaultfd test"); + return KSFT_SKIP; +} + +#endif /* __NR_userfaultfd */ diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index 5614222a6628..752ec158ac59 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -166,24 +166,22 @@ static void *vcpu_worker(void *data) pages_count += TEST_PAGES_PER_LOOP; generate_random_array(guest_array, TEST_PAGES_PER_LOOP); } else { - TEST_ASSERT(false, - "Invalid guest sync status: " - "exit_reason=%s\n", - exit_reason_str(run->exit_reason)); + TEST_FAIL("Invalid guest sync status: " + "exit_reason=%s\n", + exit_reason_str(run->exit_reason)); } } - DEBUG("Dirtied %"PRIu64" pages\n", pages_count); + pr_info("Dirtied %"PRIu64" pages\n", pages_count); return NULL; } -static void vm_dirty_log_verify(unsigned long *bmap) +static void vm_dirty_log_verify(enum vm_guest_mode mode, unsigned long *bmap) { + uint64_t step = vm_num_host_pages(mode, 1); uint64_t page; uint64_t *value_ptr; - uint64_t step = host_page_size >= guest_page_size ? 1 : - guest_page_size / host_page_size; for (page = 0; page < host_num_pages; page += step) { value_ptr = host_test_mem + page * host_page_size; @@ -252,6 +250,8 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, uint32_t vcpuid, struct kvm_vm *vm; uint64_t extra_pg_pages = extra_mem_pages / 512 * 2; + pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode)); + vm = _vm_create(mode, DEFAULT_GUEST_PHY_PAGES + extra_pg_pages, O_RDWR); kvm_vm_elf_load(vm, program_invocation_name, 0, 0); #ifdef __x86_64__ @@ -264,6 +264,10 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, uint32_t vcpuid, #define DIRTY_MEM_BITS 30 /* 1G */ #define PAGE_SHIFT_4K 12 +#ifdef USE_CLEAR_DIRTY_LOG +static u64 dirty_log_manual_caps; +#endif + static void run_test(enum vm_guest_mode mode, unsigned long iterations, unsigned long interval, uint64_t phys_offset) { @@ -289,14 +293,11 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, * case where the size is not aligned to 64 pages. */ guest_num_pages = (1ul << (DIRTY_MEM_BITS - - vm_get_page_shift(vm))) + 16; -#ifdef __s390x__ - /* Round up to multiple of 1M (segment size) */ - guest_num_pages = (guest_num_pages + 0xff) & ~0xffUL; -#endif + vm_get_page_shift(vm))) + 3; + guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages); + host_page_size = getpagesize(); - host_num_pages = (guest_num_pages * guest_page_size) / host_page_size + - !!((guest_num_pages * guest_page_size) % host_page_size); + host_num_pages = vm_num_host_pages(mode, guest_num_pages); if (!phys_offset) { guest_test_phys_mem = (vm_get_max_gfn(vm) - @@ -311,7 +312,7 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, guest_test_phys_mem &= ~((1 << 20) - 1); #endif - DEBUG("guest physical test memory offset: 0x%lx\n", guest_test_phys_mem); + pr_info("guest physical test memory offset: 0x%lx\n", guest_test_phys_mem); bmap = bitmap_alloc(host_num_pages); host_bmap_track = bitmap_alloc(host_num_pages); @@ -320,7 +321,7 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, struct kvm_enable_cap cap = {}; cap.cap = KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2; - cap.args[0] = 1; + cap.args[0] = dirty_log_manual_caps; vm_enable_cap(vm, &cap); #endif @@ -332,8 +333,7 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, KVM_MEM_LOG_DIRTY_PAGES); /* Do mapping for the dirty track memory slot */ - virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, - guest_num_pages * guest_page_size, 0); + virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages, 0); /* Cache the HVA pointer of the region */ host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem); @@ -341,9 +341,7 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, #ifdef __x86_64__ vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); #endif -#ifdef __aarch64__ ucall_init(vm, NULL); -#endif /* Export the shared variables to the guest */ sync_global_to_guest(vm, host_page_size); @@ -369,7 +367,7 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, kvm_vm_clear_dirty_log(vm, TEST_MEM_SLOT_INDEX, bmap, 0, host_num_pages); #endif - vm_dirty_log_verify(bmap); + vm_dirty_log_verify(mode, bmap); iteration++; sync_global_to_guest(vm, iteration); } @@ -378,9 +376,9 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, host_quit = true; pthread_join(vcpu_thread, NULL); - DEBUG("Total bits checked: dirty (%"PRIu64"), clear (%"PRIu64"), " - "track_next (%"PRIu64")\n", host_dirty_count, host_clear_count, - host_track_next_count); + pr_info("Total bits checked: dirty (%"PRIu64"), clear (%"PRIu64"), " + "track_next (%"PRIu64")\n", host_dirty_count, host_clear_count, + host_track_next_count); free(bmap); free(host_bmap_track); @@ -388,15 +386,14 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, kvm_vm_free(vm); } -struct vm_guest_mode_params { +struct guest_mode { bool supported; bool enabled; }; -struct vm_guest_mode_params vm_guest_mode_params[NUM_VM_MODES]; +static struct guest_mode guest_modes[NUM_VM_MODES]; -#define vm_guest_mode_params_init(mode, supported, enabled) \ -({ \ - vm_guest_mode_params[mode] = (struct vm_guest_mode_params){ supported, enabled }; \ +#define guest_mode_init(mode, supported, enabled) ({ \ + guest_modes[mode] = (struct guest_mode){ supported, enabled }; \ }) static void help(char *name) @@ -419,7 +416,7 @@ static void help(char *name) " Guest mode IDs:\n"); for (i = 0; i < NUM_VM_MODES; ++i) { printf(" %d: %s%s\n", i, vm_guest_mode_string(i), - vm_guest_mode_params[i].supported ? " (supported)" : ""); + guest_modes[i].supported ? " (supported)" : ""); } puts(""); exit(0); @@ -433,34 +430,38 @@ int main(int argc, char *argv[]) uint64_t phys_offset = 0; unsigned int mode; int opt, i; -#ifdef __aarch64__ - unsigned int host_ipa_limit; -#endif #ifdef USE_CLEAR_DIRTY_LOG - if (!kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2)) { - fprintf(stderr, "KVM_CLEAR_DIRTY_LOG not available, skipping tests\n"); + dirty_log_manual_caps = + kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2); + if (!dirty_log_manual_caps) { + print_skip("KVM_CLEAR_DIRTY_LOG not available"); exit(KSFT_SKIP); } + dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | + KVM_DIRTY_LOG_INITIALLY_SET); #endif #ifdef __x86_64__ - vm_guest_mode_params_init(VM_MODE_PXXV48_4K, true, true); + guest_mode_init(VM_MODE_PXXV48_4K, true, true); #endif #ifdef __aarch64__ - vm_guest_mode_params_init(VM_MODE_P40V48_4K, true, true); - vm_guest_mode_params_init(VM_MODE_P40V48_64K, true, true); + guest_mode_init(VM_MODE_P40V48_4K, true, true); + guest_mode_init(VM_MODE_P40V48_64K, true, true); - host_ipa_limit = kvm_check_cap(KVM_CAP_ARM_VM_IPA_SIZE); - if (host_ipa_limit >= 52) - vm_guest_mode_params_init(VM_MODE_P52V48_64K, true, true); - if (host_ipa_limit >= 48) { - vm_guest_mode_params_init(VM_MODE_P48V48_4K, true, true); - vm_guest_mode_params_init(VM_MODE_P48V48_64K, true, true); + { + unsigned int limit = kvm_check_cap(KVM_CAP_ARM_VM_IPA_SIZE); + + if (limit >= 52) + guest_mode_init(VM_MODE_P52V48_64K, true, true); + if (limit >= 48) { + guest_mode_init(VM_MODE_P48V48_4K, true, true); + guest_mode_init(VM_MODE_P48V48_64K, true, true); + } } #endif #ifdef __s390x__ - vm_guest_mode_params_init(VM_MODE_P40V48_4K, true, true); + guest_mode_init(VM_MODE_P40V48_4K, true, true); #endif while ((opt = getopt(argc, argv, "hi:I:p:m:")) != -1) { @@ -477,13 +478,13 @@ int main(int argc, char *argv[]) case 'm': if (!mode_selected) { for (i = 0; i < NUM_VM_MODES; ++i) - vm_guest_mode_params[i].enabled = false; + guest_modes[i].enabled = false; mode_selected = true; } mode = strtoul(optarg, NULL, 10); TEST_ASSERT(mode < NUM_VM_MODES, "Guest mode ID %d too big", mode); - vm_guest_mode_params[mode].enabled = true; + guest_modes[mode].enabled = true; break; case 'h': default: @@ -495,15 +496,15 @@ int main(int argc, char *argv[]) TEST_ASSERT(iterations > 2, "Iterations must be greater than two"); TEST_ASSERT(interval > 0, "Interval must be greater than zero"); - DEBUG("Test iterations: %"PRIu64", interval: %"PRIu64" (ms)\n", - iterations, interval); + pr_info("Test iterations: %"PRIu64", interval: %"PRIu64" (ms)\n", + iterations, interval); srandom(time(0)); for (i = 0; i < NUM_VM_MODES; ++i) { - if (!vm_guest_mode_params[i].enabled) + if (!guest_modes[i].enabled) continue; - TEST_ASSERT(vm_guest_mode_params[i].supported, + TEST_ASSERT(guest_modes[i].supported, "Guest mode ID %d (%s) not supported.", i, vm_guest_mode_string(i)); run_test(i, iterations, interval, phys_offset); diff --git a/tools/testing/selftests/kvm/include/evmcs.h b/tools/testing/selftests/kvm/include/evmcs.h index 4912d23844bc..d8f4d6bfe05d 100644 --- a/tools/testing/selftests/kvm/include/evmcs.h +++ b/tools/testing/selftests/kvm/include/evmcs.h @@ -16,6 +16,8 @@ #define u32 uint32_t #define u64 uint64_t +#define EVMCS_VERSION 1 + extern bool enable_evmcs; struct hv_vp_assist_page { diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index ae0d14c2540a..a99b875f50d2 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -16,7 +16,8 @@ #include "sparsebit.h" -/* Callers of kvm_util only have an incomplete/opaque description of the +/* + * Callers of kvm_util only have an incomplete/opaque description of the * structure kvm_util is using to maintain the state of a VM. */ struct kvm_vm; @@ -24,12 +25,6 @@ struct kvm_vm; typedef uint64_t vm_paddr_t; /* Virtual Machine (Guest) physical address */ typedef uint64_t vm_vaddr_t; /* Virtual Machine (Guest) virtual address */ -#ifndef NDEBUG -#define DEBUG(...) printf(__VA_ARGS__); -#else -#define DEBUG(...) -#endif - /* Minimum allocated guest virtual and physical addresses */ #define KVM_UTIL_MIN_VADDR 0x2000 @@ -84,6 +79,23 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename, uint32_t data_memslot, uint32_t pgd_memslot); void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); + +/* + * VM VCPU Dump + * + * Input Args: + * stream - Output FILE stream + * vm - Virtual Machine + * vcpuid - VCPU ID + * indent - Left margin indent amount + * + * Output Args: None + * + * Return: None + * + * Dumps the current state of the VCPU specified by @vcpuid, within the VM + * given by @vm, to the FILE stream given by @stream. + */ void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent); @@ -100,14 +112,31 @@ int _vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid, unsigned long ioctl, void *arg); void vm_ioctl(struct kvm_vm *vm, unsigned long ioctl, void *arg); void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags); +void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa); void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid); vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, uint32_t data_memslot, uint32_t pgd_memslot); void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, - size_t size, uint32_t pgd_memslot); + unsigned int npages, uint32_t pgd_memslot); void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa); void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva); vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva); + +/* + * Address Guest Virtual to Guest Physical + * + * Input Args: + * vm - Virtual Machine + * gva - VM virtual address + * + * Output Args: None + * + * Return: + * Equivalent VM physical address + * + * Returns the VM physical address of the translated VM virtual + * address given by @gva. + */ vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva); struct kvm_run *vcpu_state(struct kvm_vm *vm, uint32_t vcpuid); @@ -118,7 +147,27 @@ void vcpu_set_mp_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_mp_state *mp_state); void vcpu_regs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs); void vcpu_regs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs); + +/* + * VM VCPU Args Set + * + * Input Args: + * vm - Virtual Machine + * vcpuid - VCPU ID + * num - number of arguments + * ... - arguments, each of type uint64_t + * + * Output Args: None + * + * Return: None + * + * Sets the first @num function input registers of the VCPU with @vcpuid, + * per the C calling convention of the architecture, to the values given + * as variable args. Each of the variable args is expected to be of type + * uint64_t. The maximum @num can be is specific to the architecture. + */ void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...); + void vcpu_sregs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs); void vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, @@ -147,15 +196,57 @@ int vcpu_nested_state_set(struct kvm_vm *vm, uint32_t vcpuid, const char *exit_reason_str(unsigned int exit_reason); void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot); + +/* + * VM Virtual Page Map + * + * Input Args: + * vm - Virtual Machine + * vaddr - VM Virtual Address + * paddr - VM Physical Address + * memslot - Memory region slot for new virtual translation tables + * + * Output Args: None + * + * Return: None + * + * Within @vm, creates a virtual translation for the page starting + * at @vaddr to the page starting at @paddr. + */ void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, - uint32_t pgd_memslot); + uint32_t memslot); + vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, uint32_t memslot); vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, vm_paddr_t paddr_min, uint32_t memslot); -struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_size, +/* + * Create a VM with reasonable defaults + * + * Input Args: + * vcpuid - The id of the single VCPU to add to the VM. + * extra_mem_pages - The number of extra pages to add (this will + * decide how much extra space we will need to + * setup the page tables using memslot 0) + * guest_code - The vCPU's entry point + * + * Output Args: None + * + * Return: + * Pointer to opaque structure that describes the created VM. + */ +struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages, void *guest_code); + +/* + * Adds a vCPU with reasonable defaults (e.g. a stack) + * + * Input Args: + * vm - Virtual Machine + * vcpuid - The id of the VCPU to add to the VM. + * guest_code - The vCPU's entry point + */ void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code); bool vm_is_unrestricted_guest(struct kvm_vm *vm); @@ -164,6 +255,21 @@ unsigned int vm_get_page_size(struct kvm_vm *vm); unsigned int vm_get_page_shift(struct kvm_vm *vm); unsigned int vm_get_max_gfn(struct kvm_vm *vm); +unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size); +unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages); +unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages); +static inline unsigned int +vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages) +{ + unsigned int n; + n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages)); +#ifdef __s390x__ + /* s390 requires 1M aligned guest sizes */ + n = (n + 255) & ~255; +#endif + return n; +} + struct kvm_userspace_memory_region * kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end); diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a41db6fb7e24..5eb01bf51b86 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -19,12 +19,28 @@ #include #include "kselftest.h" +static inline int _no_printf(const char *format, ...) { return 0; } + +#ifdef DEBUG +#define pr_debug(...) printf(__VA_ARGS__) +#else +#define pr_debug(...) _no_printf(__VA_ARGS__) +#endif +#ifndef QUIET +#define pr_info(...) printf(__VA_ARGS__) +#else +#define pr_info(...) _no_printf(__VA_ARGS__) +#endif + +void print_skip(const char *fmt, ...) __attribute__((format(printf, 1, 2))); + ssize_t test_write(int fd, const void *buf, size_t count); ssize_t test_read(int fd, void *buf, size_t count); int test_seq_read(const char *path, char **bufp, size_t *sizep); void test_assert(bool exp, const char *exp_str, - const char *file, unsigned int line, const char *fmt, ...); + const char *file, unsigned int line, const char *fmt, ...) + __attribute__((format(printf, 5, 6))); #define TEST_ASSERT(e, fmt, ...) \ test_assert((e), #e, __FILE__, __LINE__, fmt, ##__VA_ARGS__) @@ -39,4 +55,14 @@ void test_assert(bool exp, const char *exp_str, #a, #b, #a, (unsigned long) __a, #b, (unsigned long) __b); \ } while (0) +#define TEST_FAIL(fmt, ...) \ + TEST_ASSERT(false, fmt, ##__VA_ARGS__) + +size_t parse_size(const char *size); + +int64_t timespec_to_ns(struct timespec ts); +struct timespec timespec_add_ns(struct timespec ts, int64_t ns); +struct timespec timespec_add(struct timespec ts1, struct timespec ts2); +struct timespec timespec_sub(struct timespec ts1, struct timespec ts2); + #endif /* SELFTEST_KVM_TEST_UTIL_H */ diff --git a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c index 6f38c3dc0d56..0299cd81b8ba 100644 --- a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c +++ b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c @@ -24,8 +24,8 @@ void test_vcpu_creation(int first_vcpu_id, int num_vcpus) struct kvm_vm *vm; int i; - printf("Testing creating %d vCPUs, with IDs %d...%d.\n", - num_vcpus, first_vcpu_id, first_vcpu_id + num_vcpus - 1); + pr_info("Testing creating %d vCPUs, with IDs %d...%d.\n", + num_vcpus, first_vcpu_id, first_vcpu_id + num_vcpus - 1); vm = vm_create(VM_MODE_DEFAULT, DEFAULT_GUEST_PHY_PAGES, O_RDWR); @@ -41,8 +41,8 @@ int main(int argc, char *argv[]) int kvm_max_vcpu_id = kvm_check_cap(KVM_CAP_MAX_VCPU_ID); int kvm_max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); - printf("KVM_CAP_MAX_VCPU_ID: %d\n", kvm_max_vcpu_id); - printf("KVM_CAP_MAX_VCPUS: %d\n", kvm_max_vcpus); + pr_info("KVM_CAP_MAX_VCPU_ID: %d\n", kvm_max_vcpu_id); + pr_info("KVM_CAP_MAX_VCPUS: %d\n", kvm_max_vcpus); /* * Upstream KVM prior to 4.8 does not support KVM_CAP_MAX_VCPU_ID. diff --git a/tools/testing/selftests/kvm/lib/aarch64/processor.c b/tools/testing/selftests/kvm/lib/aarch64/processor.c index 86036a59a668..2afa6618b396 100644 --- a/tools/testing/selftests/kvm/lib/aarch64/processor.c +++ b/tools/testing/selftests/kvm/lib/aarch64/processor.c @@ -130,7 +130,7 @@ void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8; break; default: - TEST_ASSERT(false, "Page table levels must be 2, 3, or 4"); + TEST_FAIL("Page table levels must be 2, 3, or 4"); } *ptep = paddr | 3; @@ -173,20 +173,19 @@ vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) goto unmapped_gva; break; default: - TEST_ASSERT(false, "Page table levels must be 2, 3, or 4"); + TEST_FAIL("Page table levels must be 2, 3, or 4"); } return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1)); unmapped_gva: - TEST_ASSERT(false, "No mapping for vm virtual address, " - "gva: 0x%lx", gva); + TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva); exit(1); } static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level) { -#ifdef DEBUG_VM +#ifdef DEBUG static const char * const type[] = { "", "pud", "pmd", "pte" }; uint64_t pte, *ptep; @@ -197,7 +196,7 @@ static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t p ptep = addr_gpa2hva(vm, pte); if (!*ptep) continue; - printf("%*s%s: %lx: %lx at %p\n", indent, "", type[level], pte, *ptep, ptep); + fprintf(stream, "%*s%s: %lx: %lx at %p\n", indent, "", type[level], pte, *ptep, ptep); pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level + 1); } #endif @@ -215,7 +214,7 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) ptep = addr_gpa2hva(vm, pgd); if (!*ptep) continue; - printf("%*spgd: %lx: %lx at %p\n", indent, "", pgd, *ptep, ptep); + fprintf(stream, "%*spgd: %lx: %lx at %p\n", indent, "", pgd, *ptep, ptep); pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level); } } @@ -262,11 +261,11 @@ void aarch64_vcpu_setup(struct kvm_vm *vm, int vcpuid, struct kvm_vcpu_init *ini switch (vm->mode) { case VM_MODE_P52V48_4K: - TEST_ASSERT(false, "AArch64 does not support 4K sized pages " - "with 52-bit physical address ranges"); + TEST_FAIL("AArch64 does not support 4K sized pages " + "with 52-bit physical address ranges"); case VM_MODE_PXXV48_4K: - TEST_ASSERT(false, "AArch64 does not support 4K sized pages " - "with ANY-bit physical address ranges"); + TEST_FAIL("AArch64 does not support 4K sized pages " + "with ANY-bit physical address ranges"); case VM_MODE_P52V48_64K: tcr_el1 |= 1ul << 14; /* TG0 = 64KB */ tcr_el1 |= 6ul << 32; /* IPS = 52 bits */ @@ -288,7 +287,7 @@ void aarch64_vcpu_setup(struct kvm_vm *vm, int vcpuid, struct kvm_vcpu_init *ini tcr_el1 |= 2ul << 32; /* IPS = 40 bits */ break; default: - TEST_ASSERT(false, "Unknown guest mode, mode: 0x%x", vm->mode); + TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode); } sctlr_el1 |= (1 << 0) | (1 << 2) | (1 << 12) /* M | C | I */; @@ -333,3 +332,21 @@ void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) { aarch64_vcpu_add_default(vm, vcpuid, NULL, guest_code); } + +void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...) +{ + va_list ap; + int i; + + TEST_ASSERT(num >= 1 && num <= 8, "Unsupported number of args,\n" + " num: %u\n", num); + + va_start(ap, num); + + for (i = 0; i < num; i++) { + set_reg(vm, vcpuid, ARM64_CORE_REG(regs.regs[i]), + va_arg(ap, uint64_t)); + } + + va_end(ap); +} diff --git a/tools/testing/selftests/kvm/lib/aarch64/ucall.c b/tools/testing/selftests/kvm/lib/aarch64/ucall.c index 6cd91970fbad..c8e0ec20d3bf 100644 --- a/tools/testing/selftests/kvm/lib/aarch64/ucall.c +++ b/tools/testing/selftests/kvm/lib/aarch64/ucall.c @@ -62,7 +62,7 @@ void ucall_init(struct kvm_vm *vm, void *arg) if (ucall_mmio_init(vm, start + offset)) return; } - TEST_ASSERT(false, "Can't find a ucall mmio address"); + TEST_FAIL("Can't find a ucall mmio address"); } void ucall_uninit(struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/lib/assert.c b/tools/testing/selftests/kvm/lib/assert.c index d1cf9f6e0e6b..5ebbd0d6b472 100644 --- a/tools/testing/selftests/kvm/lib/assert.c +++ b/tools/testing/selftests/kvm/lib/assert.c @@ -82,8 +82,10 @@ test_assert(bool exp, const char *exp_str, } va_end(ap); - if (errno == EACCES) - ksft_exit_skip("Access denied - Exiting.\n"); + if (errno == EACCES) { + print_skip("Access denied - Exiting"); + exit(KSFT_SKIP); + } exit(254); } diff --git a/tools/testing/selftests/kvm/lib/io.c b/tools/testing/selftests/kvm/lib/io.c index eaf351cc7e7f..fedb2a741f0b 100644 --- a/tools/testing/selftests/kvm/lib/io.c +++ b/tools/testing/selftests/kvm/lib/io.c @@ -61,9 +61,9 @@ ssize_t test_write(int fd, const void *buf, size_t count) continue; case 0: - TEST_ASSERT(false, "Unexpected EOF,\n" - " rc: %zi num_written: %zi num_left: %zu", - rc, num_written, num_left); + TEST_FAIL("Unexpected EOF,\n" + " rc: %zi num_written: %zi num_left: %zu", + rc, num_written, num_left); break; default: @@ -138,9 +138,9 @@ ssize_t test_read(int fd, void *buf, size_t count) break; case 0: - TEST_ASSERT(false, "Unexpected EOF,\n" - " rc: %zi num_read: %zi num_left: %zu", - rc, num_read, num_left); + TEST_FAIL("Unexpected EOF,\n" + " rc: %zi num_read: %zi num_left: %zu", + rc, num_read, num_left); break; default: diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index a6dd0401eb50..8a3523d4434f 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -92,7 +92,7 @@ static void vm_open(struct kvm_vm *vm, int perm) exit(KSFT_SKIP); if (!kvm_check_cap(KVM_CAP_IMMEDIATE_EXIT)) { - fprintf(stderr, "immediate_exit not available, skipping test\n"); + print_skip("immediate_exit not available"); exit(KSFT_SKIP); } @@ -113,6 +113,25 @@ const char * const vm_guest_mode_string[] = { _Static_assert(sizeof(vm_guest_mode_string)/sizeof(char *) == NUM_VM_MODES, "Missing new mode strings?"); +struct vm_guest_mode_params { + unsigned int pa_bits; + unsigned int va_bits; + unsigned int page_size; + unsigned int page_shift; +}; + +static const struct vm_guest_mode_params vm_guest_mode_params[] = { + { 52, 48, 0x1000, 12 }, + { 52, 48, 0x10000, 16 }, + { 48, 48, 0x1000, 12 }, + { 48, 48, 0x10000, 16 }, + { 40, 48, 0x1000, 12 }, + { 40, 48, 0x10000, 16 }, + { 0, 0, 0x1000, 12 }, +}; +_Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES, + "Missing new mode params?"); + /* * VM Create * @@ -136,7 +155,8 @@ struct kvm_vm *_vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) { struct kvm_vm *vm; - DEBUG("Testing guest mode: %s\n", vm_guest_mode_string(mode)); + pr_debug("%s: mode='%s' pages='%ld' perm='%d'\n", __func__, + vm_guest_mode_string(mode), phy_pages, perm); vm = calloc(1, sizeof(*vm)); TEST_ASSERT(vm != NULL, "Insufficient Memory"); @@ -144,67 +164,45 @@ struct kvm_vm *_vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) vm->mode = mode; vm->type = 0; + vm->pa_bits = vm_guest_mode_params[mode].pa_bits; + vm->va_bits = vm_guest_mode_params[mode].va_bits; + vm->page_size = vm_guest_mode_params[mode].page_size; + vm->page_shift = vm_guest_mode_params[mode].page_shift; + /* Setup mode specific traits. */ switch (vm->mode) { case VM_MODE_P52V48_4K: vm->pgtable_levels = 4; - vm->pa_bits = 52; - vm->va_bits = 48; - vm->page_size = 0x1000; - vm->page_shift = 12; break; case VM_MODE_P52V48_64K: vm->pgtable_levels = 3; - vm->pa_bits = 52; - vm->va_bits = 48; - vm->page_size = 0x10000; - vm->page_shift = 16; break; case VM_MODE_P48V48_4K: vm->pgtable_levels = 4; - vm->pa_bits = 48; - vm->va_bits = 48; - vm->page_size = 0x1000; - vm->page_shift = 12; break; case VM_MODE_P48V48_64K: vm->pgtable_levels = 3; - vm->pa_bits = 48; - vm->va_bits = 48; - vm->page_size = 0x10000; - vm->page_shift = 16; break; case VM_MODE_P40V48_4K: vm->pgtable_levels = 4; - vm->pa_bits = 40; - vm->va_bits = 48; - vm->page_size = 0x1000; - vm->page_shift = 12; break; case VM_MODE_P40V48_64K: vm->pgtable_levels = 3; - vm->pa_bits = 40; - vm->va_bits = 48; - vm->page_size = 0x10000; - vm->page_shift = 16; break; case VM_MODE_PXXV48_4K: #ifdef __x86_64__ kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits); TEST_ASSERT(vm->va_bits == 48, "Linear address width " "(%d bits) not supported", vm->va_bits); + pr_debug("Guest physical address width detected: %d\n", + vm->pa_bits); vm->pgtable_levels = 4; - vm->page_size = 0x1000; - vm->page_shift = 12; - DEBUG("Guest physical address width detected: %d\n", - vm->pa_bits); #else - TEST_ASSERT(false, "VM_MODE_PXXV48_4K not supported on " - "non-x86 platforms"); + TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms"); #endif break; default: - TEST_ASSERT(false, "Unknown guest mode, mode: 0x%x", mode); + TEST_FAIL("Unknown guest mode, mode: 0x%x", mode); } #ifdef __aarch64__ @@ -266,7 +264,7 @@ void kvm_vm_restart(struct kvm_vm *vmp, int perm) TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n" " rc: %i errno: %i\n" " slot: %u flags: 0x%x\n" - " guest_phys_addr: 0x%lx size: 0x%lx", + " guest_phys_addr: 0x%llx size: 0x%llx", ret, errno, region->region.slot, region->region.flags, region->region.guest_phys_addr, @@ -281,7 +279,7 @@ void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log) ret = ioctl(vm->fd, KVM_GET_DIRTY_LOG, &args); TEST_ASSERT(ret == 0, "%s: KVM_GET_DIRTY_LOG failed: %s", - strerror(-ret)); + __func__, strerror(-ret)); } void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log, @@ -294,7 +292,7 @@ void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log, ret = ioctl(vm->fd, KVM_CLEAR_DIRTY_LOG, &args); TEST_ASSERT(ret == 0, "%s: KVM_CLEAR_DIRTY_LOG failed: %s", - strerror(-ret)); + __func__, strerror(-ret)); } /* @@ -582,6 +580,10 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, size_t huge_page_size = KVM_UTIL_PGS_PER_HUGEPG * vm->page_size; size_t alignment; + TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages, + "Number of guest pages is not compatible with the host. " + "Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages)); + TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical " "address not on a page boundary.\n" " guest_paddr: 0x%lx vm->page_size: 0x%x", @@ -600,7 +602,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, region = (struct userspace_mem_region *) userspace_mem_region_find( vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1); if (region != NULL) - TEST_ASSERT(false, "overlapping userspace_mem_region already " + TEST_FAIL("overlapping userspace_mem_region already " "exists\n" " requested guest_paddr: 0x%lx npages: 0x%lx " "page_size: 0x%x\n" @@ -616,7 +618,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, break; } if (region != NULL) - TEST_ASSERT(false, "A mem region with the requested slot " + TEST_FAIL("A mem region with the requested slot " "already exists.\n" " requested slot: %u paddr: 0x%lx npages: 0x%lx\n" " existing slot: %u paddr: 0x%lx size: 0x%lx", @@ -720,7 +722,7 @@ memslot2region(struct kvm_vm *vm, uint32_t memslot) " requested slot: %u\n", memslot); fputs("---- vm dump ----\n", stderr); vm_dump(stderr, vm, 2); - TEST_ASSERT(false, "Mem region not found"); + TEST_FAIL("Mem region not found"); } return region; @@ -756,6 +758,36 @@ void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags) ret, errno, slot, flags); } +/* + * VM Memory Region Move + * + * Input Args: + * vm - Virtual Machine + * slot - Slot of the memory region to move + * new_gpa - Starting guest physical address + * + * Output Args: None + * + * Return: None + * + * Change the gpa of a memory region. + */ +void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa) +{ + struct userspace_mem_region *region; + int ret; + + region = memslot2region(vm, slot); + + region->region.guest_phys_addr = new_gpa; + + ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region); + + TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed\n" + "ret: %i errno: %i slot: %u new_gpa: 0x%lx", + ret, errno, slot, new_gpa); +} + /* * VCPU mmap Size * @@ -808,7 +840,7 @@ void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid) /* Confirm a vcpu with the specified id doesn't already exist. */ vcpu = vcpu_find(vm, vcpuid); if (vcpu != NULL) - TEST_ASSERT(false, "vcpu with the specified id " + TEST_FAIL("vcpu with the specified id " "already exists,\n" " requested vcpuid: %u\n" " existing vcpuid: %u state: %p", @@ -901,8 +933,7 @@ static vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, } while (pgidx_start != 0); no_va_found: - TEST_ASSERT(false, "No vaddr of specified pages available, " - "pages: 0x%lx", pages); + TEST_FAIL("No vaddr of specified pages available, pages: 0x%lx", pages); /* NOT REACHED */ return -1; @@ -982,21 +1013,21 @@ vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, * vm - Virtual Machine * vaddr - Virtuall address to map * paddr - VM Physical Address - * size - The size of the range to map + * npages - The number of pages to map * pgd_memslot - Memory region slot for new virtual translation tables * * Output Args: None * * Return: None * - * Within the VM given by vm, creates a virtual translation for the - * page range starting at vaddr to the page range starting at paddr. + * Within the VM given by @vm, creates a virtual translation for + * @npages starting at @vaddr to the page range starting at @paddr. */ void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, - size_t size, uint32_t pgd_memslot) + unsigned int npages, uint32_t pgd_memslot) { size_t page_size = vm->page_size; - size_t npages = size / page_size; + size_t size = npages * page_size; TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow"); TEST_ASSERT(paddr + size > paddr, "Paddr overflow"); @@ -1037,7 +1068,7 @@ void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa) + (gpa - region->region.guest_phys_addr)); } - TEST_ASSERT(false, "No vm physical memory at 0x%lx", gpa); + TEST_FAIL("No vm physical memory at 0x%lx", gpa); return NULL; } @@ -1071,8 +1102,7 @@ vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva) + (hva - (uintptr_t) region->host_mem)); } - TEST_ASSERT(false, "No mapping to a guest physical address, " - "hva: %p", hva); + TEST_FAIL("No mapping to a guest physical address, hva: %p", hva); return -1; } @@ -1703,3 +1733,43 @@ unsigned int vm_get_max_gfn(struct kvm_vm *vm) { return vm->max_gfn; } + +static unsigned int vm_calc_num_pages(unsigned int num_pages, + unsigned int page_shift, + unsigned int new_page_shift, + bool ceil) +{ + unsigned int n = 1 << (new_page_shift - page_shift); + + if (page_shift >= new_page_shift) + return num_pages * (1 << (page_shift - new_page_shift)); + + return num_pages / n + !!(ceil && num_pages % n); +} + +static inline int getpageshift(void) +{ + return __builtin_ffs(getpagesize()) - 1; +} + +unsigned int +vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages) +{ + return vm_calc_num_pages(num_guest_pages, + vm_guest_mode_params[mode].page_shift, + getpageshift(), true); +} + +unsigned int +vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages) +{ + return vm_calc_num_pages(num_host_pages, getpageshift(), + vm_guest_mode_params[mode].page_shift, false); +} + +unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size) +{ + unsigned int n; + n = DIV_ROUND_UP(size, vm_guest_mode_params[mode].page_size); + return vm_adjust_num_guest_pages(mode, n); +} diff --git a/tools/testing/selftests/kvm/lib/kvm_util_internal.h b/tools/testing/selftests/kvm/lib/kvm_util_internal.h index ac50c42750cf..ca56a0133127 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util_internal.h +++ b/tools/testing/selftests/kvm/lib/kvm_util_internal.h @@ -12,17 +12,6 @@ #define KVM_DEV_PATH "/dev/kvm" -#ifndef BITS_PER_BYTE -#define BITS_PER_BYTE 8 -#endif - -#ifndef BITS_PER_LONG -#define BITS_PER_LONG (BITS_PER_BYTE * sizeof(long)) -#endif - -#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) -#define BITS_TO_LONGS(nr) DIV_ROUND_UP(nr, BITS_PER_LONG) - struct userspace_mem_region { struct userspace_mem_region *next, *prev; struct kvm_userspace_memory_region region; @@ -64,8 +53,56 @@ struct kvm_vm { }; struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid); + +/* + * Virtual Translation Tables Dump + * + * Input Args: + * stream - Output FILE stream + * vm - Virtual Machine + * indent - Left margin indent amount + * + * Output Args: None + * + * Return: None + * + * Dumps to the FILE stream given by @stream, the contents of all the + * virtual translation tables for the VM given by @vm. + */ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); + +/* + * Register Dump + * + * Input Args: + * stream - Output FILE stream + * regs - Registers + * indent - Left margin indent amount + * + * Output Args: None + * + * Return: None + * + * Dumps the state of the registers given by @regs, to the FILE stream + * given by @stream. + */ void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent); + +/* + * System Register Dump + * + * Input Args: + * stream - Output FILE stream + * sregs - System registers + * indent - Left margin indent amount + * + * Output Args: None + * + * Return: None + * + * Dumps the state of the system registers given by @sregs, to the FILE stream + * given by @stream. + */ void sregs_dump(FILE *stream, struct kvm_sregs *sregs, uint8_t indent); struct userspace_mem_region * diff --git a/tools/testing/selftests/kvm/lib/s390x/processor.c b/tools/testing/selftests/kvm/lib/s390x/processor.c index 32a02360b1eb..8d94961bd046 100644 --- a/tools/testing/selftests/kvm/lib/s390x/processor.c +++ b/tools/testing/selftests/kvm/lib/s390x/processor.c @@ -51,22 +51,6 @@ static uint64_t virt_alloc_region(struct kvm_vm *vm, int ri, uint32_t memslot) | ((ri < 4 ? (PAGES_PER_REGION - 1) : 0) & REGION_ENTRY_LENGTH); } -/* - * VM Virtual Page Map - * - * Input Args: - * vm - Virtual Machine - * gva - VM Virtual Address - * gpa - VM Physical Address - * memslot - Memory region slot for new virtual translation tables - * - * Output Args: None - * - * Return: None - * - * Within the VM given by vm, creates a virtual translation for the page - * starting at vaddr to the page starting at paddr. - */ void virt_pg_map(struct kvm_vm *vm, uint64_t gva, uint64_t gpa, uint32_t memslot) { @@ -107,26 +91,6 @@ void virt_pg_map(struct kvm_vm *vm, uint64_t gva, uint64_t gpa, entry[idx] = gpa; } -/* - * Address Guest Virtual to Guest Physical - * - * Input Args: - * vm - Virtual Machine - * gpa - VM virtual address - * - * Output Args: None - * - * Return: - * Equivalent VM physical address - * - * Translates the VM virtual address given by gva to a VM physical - * address and then locates the memory region containing the VM - * physical address, within the VM given by vm. When found, the host - * virtual address providing the memory to the vm physical address is - * returned. - * A TEST_ASSERT failure occurs if no region containing translated - * VM virtual address exists. - */ vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) { int ri, idx; @@ -196,21 +160,6 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) virt_dump_region(stream, vm, indent, vm->pgd); } -/* - * Create a VM with reasonable defaults - * - * Input Args: - * vcpuid - The id of the single VCPU to add to the VM. - * extra_mem_pages - The size of extra memories to add (this will - * decide how much extra space we will need to - * setup the page tables using mem slot 0) - * guest_code - The vCPU's entry point - * - * Output Args: None - * - * Return: - * Pointer to opaque structure that describes the created VM. - */ struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages, void *guest_code) { @@ -231,13 +180,6 @@ struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages, return vm; } -/* - * Adds a vCPU with reasonable defaults (i.e. a stack and initial PSW) - * - * Input Args: - * vcpuid - The id of the VCPU to add to the VM. - * guest_code - The vCPU's entry point - */ void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) { size_t stack_size = DEFAULT_STACK_PGS * getpagesize(); @@ -269,6 +211,26 @@ void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) run->psw_addr = (uintptr_t)guest_code; } +void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...) +{ + va_list ap; + struct kvm_regs regs; + int i; + + TEST_ASSERT(num >= 1 && num <= 5, "Unsupported number of args,\n" + " num: %u\n", + num); + + va_start(ap, num); + vcpu_regs_get(vm, vcpuid, ®s); + + for (i = 0; i < num; i++) + regs.gprs[i + 2] = va_arg(ap, uint64_t); + + vcpu_regs_set(vm, vcpuid, ®s); + va_end(ap); +} + void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent) { struct vcpu *vcpu = vm->vcpu_head; diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c new file mode 100644 index 000000000000..689e97c27ee2 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * tools/testing/selftests/kvm/lib/test_util.c + * + * Copyright (C) 2020, Google LLC. + */ +#include +#include +#include +#include +#include "test_util.h" + +/* + * Parses "[0-9]+[kmgt]?". + */ +size_t parse_size(const char *size) +{ + size_t base; + char *scale; + int shift = 0; + + TEST_ASSERT(size && isdigit(size[0]), "Need at least one digit in '%s'", size); + + base = strtoull(size, &scale, 0); + + TEST_ASSERT(base != ULLONG_MAX, "Overflow parsing size!"); + + switch (tolower(*scale)) { + case 't': + shift = 40; + break; + case 'g': + shift = 30; + break; + case 'm': + shift = 20; + break; + case 'k': + shift = 10; + break; + case 'b': + case '\0': + shift = 0; + break; + default: + TEST_ASSERT(false, "Unknown size letter %c", *scale); + } + + TEST_ASSERT((base << shift) >> shift == base, "Overflow scaling size!"); + + return base << shift; +} + +int64_t timespec_to_ns(struct timespec ts) +{ + return (int64_t)ts.tv_nsec + 1000000000LL * (int64_t)ts.tv_sec; +} + +struct timespec timespec_add_ns(struct timespec ts, int64_t ns) +{ + struct timespec res; + + res.tv_nsec = ts.tv_nsec + ns; + res.tv_sec = ts.tv_sec + res.tv_nsec / 1000000000LL; + res.tv_nsec %= 1000000000LL; + + return res; +} + +struct timespec timespec_add(struct timespec ts1, struct timespec ts2) +{ + int64_t ns1 = timespec_to_ns(ts1); + int64_t ns2 = timespec_to_ns(ts2); + return timespec_add_ns((struct timespec){0}, ns1 + ns2); +} + +struct timespec timespec_sub(struct timespec ts1, struct timespec ts2) +{ + int64_t ns1 = timespec_to_ns(ts1); + int64_t ns2 = timespec_to_ns(ts2); + return timespec_add_ns((struct timespec){0}, ns1 - ns2); +} + +void print_skip(const char *fmt, ...) +{ + va_list ap; + + assert(fmt); + va_start(ap, fmt); + vprintf(fmt, ap); + va_end(ap); + puts(", skipping test"); +} diff --git a/tools/testing/selftests/kvm/lib/x86_64/processor.c b/tools/testing/selftests/kvm/lib/x86_64/processor.c index 683d3bdb8f6a..f6eb34eaa0d2 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/processor.c +++ b/tools/testing/selftests/kvm/lib/x86_64/processor.c @@ -77,20 +77,6 @@ struct pageTableEntry { uint64_t execute_disable:1; }; -/* Register Dump - * - * Input Args: - * indent - Left margin indent amount - * regs - register - * - * Output Args: - * stream - Output FILE stream - * - * Return: None - * - * Dumps the state of the registers given by regs, to the FILE stream - * given by steam. - */ void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent) { @@ -115,19 +101,20 @@ void regs_dump(FILE *stream, struct kvm_regs *regs, regs->rip, regs->rflags); } -/* Segment Dump +/* + * Segment Dump * * Input Args: - * indent - Left margin indent amount + * stream - Output FILE stream * segment - KVM segment + * indent - Left margin indent amount * - * Output Args: - * stream - Output FILE stream + * Output Args: None * * Return: None * - * Dumps the state of the KVM segment given by segment, to the FILE stream - * given by steam. + * Dumps the state of the KVM segment given by @segment, to the FILE stream + * given by @stream. */ static void segment_dump(FILE *stream, struct kvm_segment *segment, uint8_t indent) @@ -146,19 +133,20 @@ static void segment_dump(FILE *stream, struct kvm_segment *segment, segment->unusable, segment->padding); } -/* dtable Dump +/* + * dtable Dump * * Input Args: - * indent - Left margin indent amount - * dtable - KVM dtable - * - * Output Args: * stream - Output FILE stream + * dtable - KVM dtable + * indent - Left margin indent amount + * + * Output Args: None * * Return: None * - * Dumps the state of the KVM dtable given by dtable, to the FILE stream - * given by steam. + * Dumps the state of the KVM dtable given by @dtable, to the FILE stream + * given by @stream. */ static void dtable_dump(FILE *stream, struct kvm_dtable *dtable, uint8_t indent) @@ -169,20 +157,6 @@ static void dtable_dump(FILE *stream, struct kvm_dtable *dtable, dtable->padding[0], dtable->padding[1], dtable->padding[2]); } -/* System Register Dump - * - * Input Args: - * indent - Left margin indent amount - * sregs - System registers - * - * Output Args: - * stream - Output FILE stream - * - * Return: None - * - * Dumps the state of the system registers given by sregs, to the FILE stream - * given by steam. - */ void sregs_dump(FILE *stream, struct kvm_sregs *sregs, uint8_t indent) { @@ -240,21 +214,6 @@ void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot) } } -/* VM Virtual Page Map - * - * Input Args: - * vm - Virtual Machine - * vaddr - VM Virtual Address - * paddr - VM Physical Address - * pgd_memslot - Memory region slot for new virtual translation tables - * - * Output Args: None - * - * Return: None - * - * Within the VM given by vm, creates a virtual translation for the page - * starting at vaddr to the page starting at paddr. - */ void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, uint32_t pgd_memslot) { @@ -326,20 +285,6 @@ void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, pte[index[0]].present = 1; } -/* Virtual Translation Tables Dump - * - * Input Args: - * vm - Virtual Machine - * indent - Left margin indent amount - * - * Output Args: - * stream - Output FILE stream - * - * Return: None - * - * Dumps to the FILE stream given by stream, the contents of all the - * virtual translation tables for the VM given by vm. - */ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) { struct pageMapL4Entry *pml4e, *pml4e_start; @@ -421,7 +366,8 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) } } -/* Set Unusable Segment +/* + * Set Unusable Segment * * Input Args: None * @@ -430,7 +376,7 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) * * Return: None * - * Sets the segment register pointed to by segp to an unusable state. + * Sets the segment register pointed to by @segp to an unusable state. */ static void kvm_seg_set_unusable(struct kvm_segment *segp) { @@ -460,7 +406,8 @@ static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp) } -/* Set Long Mode Flat Kernel Code Segment +/* + * Set Long Mode Flat Kernel Code Segment * * Input Args: * vm - VM whose GDT is being filled, or NULL to only write segp @@ -471,8 +418,8 @@ static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp) * * Return: None * - * Sets up the KVM segment pointed to by segp, to be a code segment - * with the selector value given by selector. + * Sets up the KVM segment pointed to by @segp, to be a code segment + * with the selector value given by @selector. */ static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector, struct kvm_segment *segp) @@ -491,7 +438,8 @@ static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector, kvm_seg_fill_gdt_64bit(vm, segp); } -/* Set Long Mode Flat Kernel Data Segment +/* + * Set Long Mode Flat Kernel Data Segment * * Input Args: * vm - VM whose GDT is being filled, or NULL to only write segp @@ -502,8 +450,8 @@ static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector, * * Return: None * - * Sets up the KVM segment pointed to by segp, to be a data segment - * with the selector value given by selector. + * Sets up the KVM segment pointed to by @segp, to be a data segment + * with the selector value given by @selector. */ static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector, struct kvm_segment *segp) @@ -521,24 +469,6 @@ static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector, kvm_seg_fill_gdt_64bit(vm, segp); } -/* Address Guest Virtual to Guest Physical - * - * Input Args: - * vm - Virtual Machine - * gpa - VM virtual address - * - * Output Args: None - * - * Return: - * Equivalent VM physical address - * - * Translates the VM virtual address given by gva to a VM physical - * address and then locates the memory region containing the VM - * physical address, within the VM given by vm. When found, the host - * virtual address providing the memory to the vm physical address is returned. - * A TEST_ASSERT failure occurs if no region containing translated - * VM virtual address exists. - */ vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) { uint16_t index[4]; @@ -576,8 +506,7 @@ vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) return (pte[index[0]].address * vm->page_size) + (gva & 0xfffu); unmapped_gva: - TEST_ASSERT(false, "No mapping for vm virtual address, " - "gva: 0x%lx", gva); + TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva); exit(EXIT_FAILURE); } @@ -634,18 +563,13 @@ static void vcpu_setup(struct kvm_vm *vm, int vcpuid, int pgd_memslot, int gdt_m break; default: - TEST_ASSERT(false, "Unknown guest mode, mode: 0x%x", vm->mode); + TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode); } sregs.cr3 = vm->pgd; vcpu_sregs_set(vm, vcpuid, &sregs); } -/* Adds a vCPU with reasonable defaults (i.e., a stack) - * - * Input Args: - * vcpuid - The id of the VCPU to add to the VM. - * guest_code - The vCPU's entry point - */ + void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) { struct kvm_mp_state mp_state; @@ -670,7 +594,8 @@ void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) vcpu_set_mp_state(vm, vcpuid, &mp_state); } -/* Allocate an instance of struct kvm_cpuid2 +/* + * Allocate an instance of struct kvm_cpuid2 * * Input Args: None * @@ -703,7 +628,8 @@ static struct kvm_cpuid2 *allocate_kvm_cpuid2(void) return cpuid; } -/* KVM Supported CPUID Get +/* + * KVM Supported CPUID Get * * Input Args: None * @@ -735,11 +661,12 @@ struct kvm_cpuid2 *kvm_get_supported_cpuid(void) return cpuid; } -/* Locate a cpuid entry. +/* + * Locate a cpuid entry. * * Input Args: - * cpuid: The cpuid. * function: The function of the cpuid entry to find. + * index: The index of the cpuid entry. * * Output Args: None * @@ -766,7 +693,8 @@ kvm_get_supported_cpuid_index(uint32_t function, uint32_t index) return entry; } -/* VM VCPU CPUID Set +/* + * VM VCPU CPUID Set * * Input Args: * vm - Virtual Machine @@ -793,20 +721,6 @@ void vcpu_set_cpuid(struct kvm_vm *vm, } -/* Create a VM with reasonable defaults - * - * Input Args: - * vcpuid - The id of the single VCPU to add to the VM. - * extra_mem_pages - The size of extra memories to add (this will - * decide how much extra space we will need to - * setup the page tables using mem slot 0) - * guest_code - The vCPU's entry point - * - * Output Args: None - * - * Return: - * Pointer to opaque structure that describes the created VM. - */ struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages, void *guest_code) { @@ -837,7 +751,8 @@ struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages, return vm; } -/* VCPU Get MSR +/* + * VCPU Get MSR * * Input Args: * vm - Virtual Machine @@ -869,7 +784,8 @@ uint64_t vcpu_get_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index) return buffer.entry.data; } -/* _VCPU Set MSR +/* + * _VCPU Set MSR * * Input Args: * vm - Virtual Machine @@ -902,7 +818,8 @@ int _vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index, return r; } -/* VCPU Set MSR +/* + * VCPU Set MSR * * Input Args: * vm - Virtual Machine @@ -926,22 +843,6 @@ void vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index, " rc: %i errno: %i", r, errno); } -/* VM VCPU Args Set - * - * Input Args: - * vm - Virtual Machine - * vcpuid - VCPU ID - * num - number of arguments - * ... - arguments, each of type uint64_t - * - * Output Args: None - * - * Return: None - * - * Sets the first num function input arguments to the values - * given as variable args. Each of the variable args is expected to - * be of type uint64_t. - */ void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...) { va_list ap; @@ -976,22 +877,6 @@ void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...) va_end(ap); } -/* - * VM VCPU Dump - * - * Input Args: - * vm - Virtual Machine - * vcpuid - VCPU ID - * indent - Left margin indent amount - * - * Output Args: - * stream - Output FILE stream - * - * Return: None - * - * Dumps the current state of the VCPU specified by vcpuid, within the VM - * given by vm, to the FILE stream given by stream. - */ void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent) { struct kvm_regs regs; diff --git a/tools/testing/selftests/kvm/lib/x86_64/svm.c b/tools/testing/selftests/kvm/lib/x86_64/svm.c index 6e05a8fc3fe0..c42401068373 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/svm.c +++ b/tools/testing/selftests/kvm/lib/x86_64/svm.c @@ -154,7 +154,7 @@ void nested_svm_check_supported(void) kvm_get_supported_cpuid_entry(0x80000001); if (!(entry->ecx & CPUID_SVM)) { - fprintf(stderr, "nested SVM not enabled, skipping test\n"); + print_skip("nested SVM not enabled"); exit(KSFT_SKIP); } } diff --git a/tools/testing/selftests/kvm/lib/x86_64/vmx.c b/tools/testing/selftests/kvm/lib/x86_64/vmx.c index 7aaa99ca4dbc..6f17f69394be 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86_64/vmx.c @@ -191,7 +191,7 @@ bool load_vmcs(struct vmx_pages *vmx) if (evmcs_vmptrld(vmx->enlightened_vmcs_gpa, vmx->enlightened_vmcs)) return false; - current_evmcs->revision_id = vmcs_revision(); + current_evmcs->revision_id = EVMCS_VERSION; } return true; @@ -381,7 +381,7 @@ void nested_vmx_check_supported(void) struct kvm_cpuid_entry2 *entry = kvm_get_supported_cpuid_entry(1); if (!(entry->ecx & CPUID_VMX)) { - fprintf(stderr, "nested VMX not enabled, skipping test\n"); + print_skip("nested VMX not enabled"); exit(KSFT_SKIP); } } diff --git a/tools/testing/selftests/kvm/s390x/memop.c b/tools/testing/selftests/kvm/s390x/memop.c index 9edaa9a134ce..9f49ead380ab 100644 --- a/tools/testing/selftests/kvm/s390x/memop.c +++ b/tools/testing/selftests/kvm/s390x/memop.c @@ -40,7 +40,7 @@ int main(int argc, char *argv[]) maxsize = kvm_check_cap(KVM_CAP_S390_MEM_OP); if (!maxsize) { - fprintf(stderr, "CAP_S390_MEM_OP not supported -> skip test\n"); + print_skip("CAP_S390_MEM_OP not supported"); exit(KSFT_SKIP); } if (maxsize > sizeof(mem1)) diff --git a/tools/testing/selftests/kvm/s390x/resets.c b/tools/testing/selftests/kvm/s390x/resets.c index 1485bc6c8999..b143db6d8693 100644 --- a/tools/testing/selftests/kvm/s390x/resets.c +++ b/tools/testing/selftests/kvm/s390x/resets.c @@ -20,29 +20,42 @@ struct kvm_s390_irq buf[VCPU_ID + LOCAL_IRQS]; struct kvm_vm *vm; struct kvm_run *run; -struct kvm_sync_regs *regs; -static uint64_t regs_null[16]; - -static uint64_t crs[16] = { 0x40000ULL, - 0x42000ULL, - 0, 0, 0, 0, 0, - 0x43000ULL, - 0, 0, 0, 0, 0, - 0x44000ULL, - 0, 0 -}; +struct kvm_sync_regs *sync_regs; +static uint8_t regs_null[512]; static void guest_code_initial(void) { - /* Round toward 0 */ - uint32_t fpc = 0x11; + /* set several CRs to "safe" value */ + unsigned long cr2_59 = 0x10; /* enable guarded storage */ + unsigned long cr8_63 = 0x1; /* monitor mask = 1 */ + unsigned long cr10 = 1; /* PER START */ + unsigned long cr11 = -1; /* PER END */ + /* Dirty registers */ asm volatile ( - " lctlg 0,15,%0\n" - " sfpc %1\n" - : : "Q" (crs), "d" (fpc)); - GUEST_SYNC(0); + " lghi 2,0x11\n" /* Round toward 0 */ + " sfpc 2\n" /* set fpc to !=0 */ + " lctlg 2,2,%0\n" + " lctlg 8,8,%1\n" + " lctlg 10,10,%2\n" + " lctlg 11,11,%3\n" + /* now clobber some general purpose regs */ + " llihh 0,0xffff\n" + " llihl 1,0x5555\n" + " llilh 2,0xaaaa\n" + " llill 3,0x0000\n" + /* now clobber a floating point reg */ + " lghi 4,0x1\n" + " cdgbr 0,4\n" + /* now clobber an access reg */ + " sar 9,4\n" + /* We embed diag 501 here to control register content */ + " diag 0,0,0x501\n" + : + : "m" (cr2_59), "m" (cr8_63), "m" (cr10), "m" (cr11) + /* no clobber list as this should not return */ + ); } static void test_one_reg(uint64_t id, uint64_t value) @@ -53,7 +66,7 @@ static void test_one_reg(uint64_t id, uint64_t value) reg.addr = (uintptr_t)&eval_reg; reg.id = id; vcpu_get_reg(vm, VCPU_ID, ®); - TEST_ASSERT(eval_reg == value, "value == %s", value); + TEST_ASSERT(eval_reg == value, "value == 0x%lx", value); } static void assert_noirq(void) @@ -87,6 +100,31 @@ static void assert_clear(void) vcpu_fpu_get(vm, VCPU_ID, &fpu); TEST_ASSERT(!memcmp(&fpu.fprs, regs_null, sizeof(fpu.fprs)), "fprs == 0"); + + /* sync regs */ + TEST_ASSERT(!memcmp(sync_regs->gprs, regs_null, sizeof(sync_regs->gprs)), + "gprs0-15 == 0 (sync_regs)"); + + TEST_ASSERT(!memcmp(sync_regs->acrs, regs_null, sizeof(sync_regs->acrs)), + "acrs0-15 == 0 (sync_regs)"); + + TEST_ASSERT(!memcmp(sync_regs->vrs, regs_null, sizeof(sync_regs->vrs)), + "vrs0-15 == 0 (sync_regs)"); +} + +static void assert_initial_noclear(void) +{ + TEST_ASSERT(sync_regs->gprs[0] == 0xffff000000000000UL, + "gpr0 == 0xffff000000000000 (sync_regs)"); + TEST_ASSERT(sync_regs->gprs[1] == 0x0000555500000000UL, + "gpr1 == 0x0000555500000000 (sync_regs)"); + TEST_ASSERT(sync_regs->gprs[2] == 0x00000000aaaa0000UL, + "gpr2 == 0x00000000aaaa0000 (sync_regs)"); + TEST_ASSERT(sync_regs->gprs[3] == 0x0000000000000000UL, + "gpr3 == 0x0000000000000000 (sync_regs)"); + TEST_ASSERT(sync_regs->fprs[0] == 0x3ff0000000000000UL, + "fpr0 == 0f1 (sync_regs)"); + TEST_ASSERT(sync_regs->acrs[9] == 1, "ar9 == 1 (sync_regs)"); } static void assert_initial(void) @@ -94,12 +132,32 @@ static void assert_initial(void) struct kvm_sregs sregs; struct kvm_fpu fpu; + /* KVM_GET_SREGS */ vcpu_sregs_get(vm, VCPU_ID, &sregs); - TEST_ASSERT(sregs.crs[0] == 0xE0UL, "cr0 == 0xE0"); - TEST_ASSERT(sregs.crs[14] == 0xC2000000UL, "cr14 == 0xC2000000"); + TEST_ASSERT(sregs.crs[0] == 0xE0UL, "cr0 == 0xE0 (KVM_GET_SREGS)"); + TEST_ASSERT(sregs.crs[14] == 0xC2000000UL, + "cr14 == 0xC2000000 (KVM_GET_SREGS)"); TEST_ASSERT(!memcmp(&sregs.crs[1], regs_null, sizeof(sregs.crs[1]) * 12), - "cr1-13 == 0"); - TEST_ASSERT(sregs.crs[15] == 0, "cr15 == 0"); + "cr1-13 == 0 (KVM_GET_SREGS)"); + TEST_ASSERT(sregs.crs[15] == 0, "cr15 == 0 (KVM_GET_SREGS)"); + + /* sync regs */ + TEST_ASSERT(sync_regs->crs[0] == 0xE0UL, "cr0 == 0xE0 (sync_regs)"); + TEST_ASSERT(sync_regs->crs[14] == 0xC2000000UL, + "cr14 == 0xC2000000 (sync_regs)"); + TEST_ASSERT(!memcmp(&sync_regs->crs[1], regs_null, 8 * 12), + "cr1-13 == 0 (sync_regs)"); + TEST_ASSERT(sync_regs->crs[15] == 0, "cr15 == 0 (sync_regs)"); + TEST_ASSERT(sync_regs->fpc == 0, "fpc == 0 (sync_regs)"); + TEST_ASSERT(sync_regs->todpr == 0, "todpr == 0 (sync_regs)"); + TEST_ASSERT(sync_regs->cputm == 0, "cputm == 0 (sync_regs)"); + TEST_ASSERT(sync_regs->ckc == 0, "ckc == 0 (sync_regs)"); + TEST_ASSERT(sync_regs->pp == 0, "pp == 0 (sync_regs)"); + TEST_ASSERT(sync_regs->gbea == 1, "gbea == 1 (sync_regs)"); + + /* kvm_run */ + TEST_ASSERT(run->psw_addr == 0, "psw_addr == 0 (kvm_run)"); + TEST_ASSERT(run->psw_mask == 0, "psw_mask == 0 (kvm_run)"); vcpu_fpu_get(vm, VCPU_ID, &fpu); TEST_ASSERT(!fpu.fpc, "fpc == 0"); @@ -111,9 +169,19 @@ static void assert_initial(void) test_one_reg(KVM_REG_S390_CLOCK_COMP, 0); } +static void assert_normal_noclear(void) +{ + TEST_ASSERT(sync_regs->crs[2] == 0x10, "cr2 == 10 (sync_regs)"); + TEST_ASSERT(sync_regs->crs[8] == 1, "cr10 == 1 (sync_regs)"); + TEST_ASSERT(sync_regs->crs[10] == 1, "cr10 == 1 (sync_regs)"); + TEST_ASSERT(sync_regs->crs[11] == -1, "cr11 == -1 (sync_regs)"); +} + static void assert_normal(void) { test_one_reg(KVM_REG_S390_PFTOKEN, KVM_S390_PFAULT_TOKEN_INVALID); + TEST_ASSERT(sync_regs->pft == KVM_S390_PFAULT_TOKEN_INVALID, + "pft == 0xff..... (sync_regs)"); assert_noirq(); } @@ -134,53 +202,67 @@ static void inject_irq(int cpu_id) static void test_normal(void) { - printf("Testing normal reset\n"); + pr_info("Testing normal reset\n"); /* Create VM */ vm = vm_create_default(VCPU_ID, 0, guest_code_initial); run = vcpu_state(vm, VCPU_ID); - regs = &run->s.regs; + sync_regs = &run->s.regs; vcpu_run(vm, VCPU_ID); inject_irq(VCPU_ID); vcpu_ioctl(vm, VCPU_ID, KVM_S390_NORMAL_RESET, 0); + + /* must clears */ assert_normal(); + /* must not clears */ + assert_normal_noclear(); + assert_initial_noclear(); + kvm_vm_free(vm); } static void test_initial(void) { - printf("Testing initial reset\n"); + pr_info("Testing initial reset\n"); vm = vm_create_default(VCPU_ID, 0, guest_code_initial); run = vcpu_state(vm, VCPU_ID); - regs = &run->s.regs; + sync_regs = &run->s.regs; vcpu_run(vm, VCPU_ID); inject_irq(VCPU_ID); vcpu_ioctl(vm, VCPU_ID, KVM_S390_INITIAL_RESET, 0); + + /* must clears */ assert_normal(); assert_initial(); + /* must not clears */ + assert_initial_noclear(); + kvm_vm_free(vm); } static void test_clear(void) { - printf("Testing clear reset\n"); + pr_info("Testing clear reset\n"); vm = vm_create_default(VCPU_ID, 0, guest_code_initial); run = vcpu_state(vm, VCPU_ID); - regs = &run->s.regs; + sync_regs = &run->s.regs; vcpu_run(vm, VCPU_ID); inject_irq(VCPU_ID); vcpu_ioctl(vm, VCPU_ID, KVM_S390_CLEAR_RESET, 0); + + /* must clears */ assert_normal(); assert_initial(); assert_clear(); + kvm_vm_free(vm); } diff --git a/tools/testing/selftests/kvm/s390x/sync_regs_test.c b/tools/testing/selftests/kvm/s390x/sync_regs_test.c index b705637ca14b..5731ccf34917 100644 --- a/tools/testing/selftests/kvm/s390x/sync_regs_test.c +++ b/tools/testing/selftests/kvm/s390x/sync_regs_test.c @@ -42,6 +42,13 @@ static void guest_code(void) " values did not match: 0x%llx, 0x%llx\n", \ left->reg, right->reg) +#define REG_COMPARE32(reg) \ + TEST_ASSERT(left->reg == right->reg, \ + "Register " #reg \ + " values did not match: 0x%x, 0x%x\n", \ + left->reg, right->reg) + + static void compare_regs(struct kvm_regs *left, struct kvm_sync_regs *right) { int i; @@ -55,7 +62,7 @@ static void compare_sregs(struct kvm_sregs *left, struct kvm_sync_regs *right) int i; for (i = 0; i < 16; i++) - REG_COMPARE(acrs[i]); + REG_COMPARE32(acrs[i]); for (i = 0; i < 16; i++) REG_COMPARE(crs[i]); @@ -79,7 +86,7 @@ int main(int argc, char *argv[]) cap = kvm_check_cap(KVM_CAP_SYNC_REGS); if (!cap) { - fprintf(stderr, "CAP_SYNC_REGS not supported, skipping test\n"); + print_skip("CAP_SYNC_REGS not supported"); exit(KSFT_SKIP); } @@ -155,7 +162,7 @@ int main(int argc, char *argv[]) "r11 sync regs value incorrect 0x%llx.", run->s.regs.gprs[11]); TEST_ASSERT(run->s.regs.acrs[0] == 1 << 11, - "acr0 sync regs value incorrect 0x%llx.", + "acr0 sync regs value incorrect 0x%x.", run->s.regs.acrs[0]); vcpu_regs_get(vm, VCPU_ID, ®s); diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c new file mode 100644 index 000000000000..fcc840088c91 --- /dev/null +++ b/tools/testing/selftests/kvm/steal_time.c @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * steal/stolen time test + * + * Copyright (C) 2020, Red Hat, Inc. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" + +#define NR_VCPUS 4 +#define ST_GPA_BASE (1 << 30) +#define MIN_RUN_DELAY_NS 200000UL + +static void *st_gva[NR_VCPUS]; +static uint64_t guest_stolen_time[NR_VCPUS]; + +#if defined(__x86_64__) + +/* steal_time must have 64-byte alignment */ +#define STEAL_TIME_SIZE ((sizeof(struct kvm_steal_time) + 63) & ~63) + +static void check_status(struct kvm_steal_time *st) +{ + GUEST_ASSERT(!(READ_ONCE(st->version) & 1)); + GUEST_ASSERT(READ_ONCE(st->flags) == 0); + GUEST_ASSERT(READ_ONCE(st->preempted) == 0); +} + +static void guest_code(int cpu) +{ + struct kvm_steal_time *st = st_gva[cpu]; + uint32_t version; + + GUEST_ASSERT(rdmsr(MSR_KVM_STEAL_TIME) == ((uint64_t)st_gva[cpu] | KVM_MSR_ENABLED)); + + memset(st, 0, sizeof(*st)); + GUEST_SYNC(0); + + check_status(st); + WRITE_ONCE(guest_stolen_time[cpu], st->steal); + version = READ_ONCE(st->version); + check_status(st); + GUEST_SYNC(1); + + check_status(st); + GUEST_ASSERT(version < READ_ONCE(st->version)); + WRITE_ONCE(guest_stolen_time[cpu], st->steal); + check_status(st); + GUEST_DONE(); +} + +static void steal_time_init(struct kvm_vm *vm) +{ + int i; + + if (!(kvm_get_supported_cpuid_entry(KVM_CPUID_FEATURES)->eax & + KVM_FEATURE_STEAL_TIME)) { + print_skip("steal-time not supported"); + exit(KSFT_SKIP); + } + + for (i = 0; i < NR_VCPUS; ++i) { + int ret; + + vcpu_set_cpuid(vm, i, kvm_get_supported_cpuid()); + + /* ST_GPA_BASE is identity mapped */ + st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE); + sync_global_to_guest(vm, st_gva[i]); + + ret = _vcpu_set_msr(vm, i, MSR_KVM_STEAL_TIME, (ulong)st_gva[i] | KVM_STEAL_RESERVED_MASK); + TEST_ASSERT(ret == 0, "Bad GPA didn't fail"); + + vcpu_set_msr(vm, i, MSR_KVM_STEAL_TIME, (ulong)st_gva[i] | KVM_MSR_ENABLED); + } +} + +static void steal_time_dump(struct kvm_vm *vm, uint32_t vcpuid) +{ + struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpuid]); + int i; + + pr_info("VCPU%d:\n", vcpuid); + pr_info(" steal: %lld\n", st->steal); + pr_info(" version: %d\n", st->version); + pr_info(" flags: %d\n", st->flags); + pr_info(" preempted: %d\n", st->preempted); + pr_info(" u8_pad: "); + for (i = 0; i < 3; ++i) + pr_info("%d", st->u8_pad[i]); + pr_info("\n pad: "); + for (i = 0; i < 11; ++i) + pr_info("%d", st->pad[i]); + pr_info("\n"); +} + +#elif defined(__aarch64__) + +/* PV_TIME_ST must have 64-byte alignment */ +#define STEAL_TIME_SIZE ((sizeof(struct st_time) + 63) & ~63) + +#define SMCCC_ARCH_FEATURES 0x80000001 +#define PV_TIME_FEATURES 0xc5000020 +#define PV_TIME_ST 0xc5000021 + +struct st_time { + uint32_t rev; + uint32_t attr; + uint64_t st_time; +}; + +static int64_t smccc(uint32_t func, uint32_t arg) +{ + unsigned long ret; + + asm volatile( + "mov x0, %1\n" + "mov x1, %2\n" + "hvc #0\n" + "mov %0, x0\n" + : "=r" (ret) : "r" (func), "r" (arg) : + "x0", "x1", "x2", "x3"); + + return ret; +} + +static void check_status(struct st_time *st) +{ + GUEST_ASSERT(READ_ONCE(st->rev) == 0); + GUEST_ASSERT(READ_ONCE(st->attr) == 0); +} + +static void guest_code(int cpu) +{ + struct st_time *st; + int64_t status; + + status = smccc(SMCCC_ARCH_FEATURES, PV_TIME_FEATURES); + GUEST_ASSERT(status == 0); + status = smccc(PV_TIME_FEATURES, PV_TIME_FEATURES); + GUEST_ASSERT(status == 0); + status = smccc(PV_TIME_FEATURES, PV_TIME_ST); + GUEST_ASSERT(status == 0); + + status = smccc(PV_TIME_ST, 0); + GUEST_ASSERT(status != -1); + GUEST_ASSERT(status == (ulong)st_gva[cpu]); + + st = (struct st_time *)status; + GUEST_SYNC(0); + + check_status(st); + WRITE_ONCE(guest_stolen_time[cpu], st->st_time); + GUEST_SYNC(1); + + check_status(st); + WRITE_ONCE(guest_stolen_time[cpu], st->st_time); + GUEST_DONE(); +} + +static void steal_time_init(struct kvm_vm *vm) +{ + struct kvm_device_attr dev = { + .group = KVM_ARM_VCPU_PVTIME_CTRL, + .attr = KVM_ARM_VCPU_PVTIME_IPA, + }; + int i, ret; + + ret = _vcpu_ioctl(vm, 0, KVM_HAS_DEVICE_ATTR, &dev); + if (ret != 0 && errno == ENXIO) { + print_skip("steal-time not supported"); + exit(KSFT_SKIP); + } + + for (i = 0; i < NR_VCPUS; ++i) { + uint64_t st_ipa; + + vcpu_ioctl(vm, i, KVM_HAS_DEVICE_ATTR, &dev); + + dev.addr = (uint64_t)&st_ipa; + + /* ST_GPA_BASE is identity mapped */ + st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE); + sync_global_to_guest(vm, st_gva[i]); + + st_ipa = (ulong)st_gva[i] | 1; + ret = _vcpu_ioctl(vm, i, KVM_SET_DEVICE_ATTR, &dev); + TEST_ASSERT(ret == -1 && errno == EINVAL, "Bad IPA didn't report EINVAL"); + + st_ipa = (ulong)st_gva[i]; + vcpu_ioctl(vm, i, KVM_SET_DEVICE_ATTR, &dev); + + ret = _vcpu_ioctl(vm, i, KVM_SET_DEVICE_ATTR, &dev); + TEST_ASSERT(ret == -1 && errno == EEXIST, "Set IPA twice without EEXIST"); + + } +} + +static void steal_time_dump(struct kvm_vm *vm, uint32_t vcpuid) +{ + struct st_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpuid]); + + pr_info("VCPU%d:\n", vcpuid); + pr_info(" rev: %d\n", st->rev); + pr_info(" attr: %d\n", st->attr); + pr_info(" st_time: %ld\n", st->st_time); +} + +#endif + +static long get_run_delay(void) +{ + char path[64]; + long val[2]; + FILE *fp; + + sprintf(path, "/proc/%ld/schedstat", syscall(SYS_gettid)); + fp = fopen(path, "r"); + fscanf(fp, "%ld %ld ", &val[0], &val[1]); + fclose(fp); + + return val[1]; +} + +static void *do_steal_time(void *arg) +{ + struct timespec ts, stop; + + clock_gettime(CLOCK_MONOTONIC, &ts); + stop = timespec_add_ns(ts, MIN_RUN_DELAY_NS); + + while (1) { + clock_gettime(CLOCK_MONOTONIC, &ts); + if (timespec_to_ns(timespec_sub(ts, stop)) >= 0) + break; + } + + return NULL; +} + +static void run_vcpu(struct kvm_vm *vm, uint32_t vcpuid) +{ + struct ucall uc; + + vcpu_args_set(vm, vcpuid, 1, vcpuid); + + vcpu_ioctl(vm, vcpuid, KVM_RUN, NULL); + + switch (get_ucall(vm, vcpuid, &uc)) { + case UCALL_SYNC: + case UCALL_DONE: + break; + case UCALL_ABORT: + TEST_ASSERT(false, "%s at %s:%ld", (const char *)uc.args[0], + __FILE__, uc.args[1]); + default: + TEST_ASSERT(false, "Unexpected exit: %s", + exit_reason_str(vcpu_state(vm, vcpuid)->exit_reason)); + } +} + +int main(int ac, char **av) +{ + struct kvm_vm *vm; + pthread_attr_t attr; + pthread_t thread; + cpu_set_t cpuset; + unsigned int gpages; + long stolen_time; + long run_delay; + bool verbose; + int i; + + verbose = ac > 1 && (!strncmp(av[1], "-v", 3) || !strncmp(av[1], "--verbose", 10)); + + /* Set CPU affinity so we can force preemption of the VCPU */ + CPU_ZERO(&cpuset); + CPU_SET(0, &cpuset); + pthread_attr_init(&attr); + pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + + /* Create a one VCPU guest and an identity mapped memslot for the steal time structure */ + vm = vm_create_default(0, 0, guest_code); + gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE * NR_VCPUS); + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, gpages, 0); + virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, gpages, 0); + ucall_init(vm, NULL); + + /* Add the rest of the VCPUs */ + for (i = 1; i < NR_VCPUS; ++i) + vm_vcpu_add_default(vm, i, guest_code); + + steal_time_init(vm); + + /* Run test on each VCPU */ + for (i = 0; i < NR_VCPUS; ++i) { + /* First VCPU run initializes steal-time */ + run_vcpu(vm, i); + + /* Second VCPU run, expect guest stolen time to be <= run_delay */ + run_vcpu(vm, i); + sync_global_from_guest(vm, guest_stolen_time[i]); + stolen_time = guest_stolen_time[i]; + run_delay = get_run_delay(); + TEST_ASSERT(stolen_time <= run_delay, + "Expected stolen time <= %ld, got %ld", + run_delay, stolen_time); + + /* Steal time from the VCPU. The steal time thread has the same CPU affinity as the VCPUs. */ + run_delay = get_run_delay(); + pthread_create(&thread, &attr, do_steal_time, NULL); + do + pthread_yield(); + while (get_run_delay() - run_delay < MIN_RUN_DELAY_NS); + pthread_join(thread, NULL); + run_delay = get_run_delay() - run_delay; + TEST_ASSERT(run_delay >= MIN_RUN_DELAY_NS, + "Expected run_delay >= %ld, got %ld", + MIN_RUN_DELAY_NS, run_delay); + + /* Run VCPU again to confirm stolen time is consistent with run_delay */ + run_vcpu(vm, i); + sync_global_from_guest(vm, guest_stolen_time[i]); + stolen_time = guest_stolen_time[i] - stolen_time; + TEST_ASSERT(stolen_time >= run_delay, + "Expected stolen time >= %ld, got %ld", + run_delay, stolen_time); + + if (verbose) { + pr_info("VCPU%d: total-stolen-time=%ld test-stolen-time=%ld", i, + guest_stolen_time[i], stolen_time); + if (stolen_time == run_delay) + pr_info(" (BONUS: guest test-stolen-time even exactly matches test-run_delay)"); + pr_info("\n"); + steal_time_dump(vm, i); + } + } + + return 0; +} diff --git a/tools/testing/selftests/kvm/x86_64/cr4_cpuid_sync_test.c b/tools/testing/selftests/kvm/x86_64/cr4_cpuid_sync_test.c index 63cc9c3f5ab6..140e91901582 100644 --- a/tools/testing/selftests/kvm/x86_64/cr4_cpuid_sync_test.c +++ b/tools/testing/selftests/kvm/x86_64/cr4_cpuid_sync_test.c @@ -72,7 +72,7 @@ int main(int argc, char *argv[]) entry = kvm_get_supported_cpuid_entry(1); if (!(entry->ecx & X86_FEATURE_XSAVE)) { - printf("XSAVE feature not supported, skipping test\n"); + print_skip("XSAVE feature not supported"); return 0; } @@ -101,12 +101,12 @@ int main(int argc, char *argv[]) vcpu_sregs_set(vm, VCPU_ID, &sregs); break; case UCALL_ABORT: - TEST_ASSERT(false, "Guest CR4 bit (OSXSAVE) unsynchronized with CPUID bit."); + TEST_FAIL("Guest CR4 bit (OSXSAVE) unsynchronized with CPUID bit."); break; case UCALL_DONE: goto done; default: - TEST_ASSERT(false, "Unknown ucall 0x%x.", uc.cmd); + TEST_FAIL("Unknown ucall %lu", uc.cmd); } } diff --git a/tools/testing/selftests/kvm/x86_64/evmcs_test.c b/tools/testing/selftests/kvm/x86_64/evmcs_test.c index 92915e6408e7..e6e62e5e75b2 100644 --- a/tools/testing/selftests/kvm/x86_64/evmcs_test.c +++ b/tools/testing/selftests/kvm/x86_64/evmcs_test.c @@ -21,10 +21,10 @@ void l2_guest_code(void) { - GUEST_SYNC(6); - GUEST_SYNC(7); + GUEST_SYNC(8); + /* Done, exit to L1 and never come back. */ vmcall(); } @@ -50,12 +50,17 @@ void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_SYNC(5); GUEST_ASSERT(vmptrstz() == vmx_pages->enlightened_vmcs_gpa); + current_evmcs->revision_id = -1u; + GUEST_ASSERT(vmlaunch()); + current_evmcs->revision_id = EVMCS_VERSION; + GUEST_SYNC(6); + GUEST_ASSERT(!vmlaunch()); GUEST_ASSERT(vmptrstz() == vmx_pages->enlightened_vmcs_gpa); - GUEST_SYNC(8); + GUEST_SYNC(9); GUEST_ASSERT(!vmresume()); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - GUEST_SYNC(9); + GUEST_SYNC(10); } void guest_code(struct vmx_pages *vmx_pages) @@ -67,6 +72,10 @@ void guest_code(struct vmx_pages *vmx_pages) l1_guest_code(vmx_pages); GUEST_DONE(); + + /* Try enlightened vmptrld with an incorrect GPA */ + evmcs_vmptrld(0xdeadbeef, vmx_pages->enlightened_vmcs); + GUEST_ASSERT(vmlaunch()); } int main(int argc, char *argv[]) @@ -87,7 +96,7 @@ int main(int argc, char *argv[]) if (!kvm_check_cap(KVM_CAP_NESTED_STATE) || !kvm_check_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)) { - printf("capabilities not available, skipping test\n"); + print_skip("capabilities not available"); exit(KSFT_SKIP); } @@ -109,20 +118,20 @@ int main(int argc, char *argv[]) switch (get_ucall(vm, VCPU_ID, &uc)) { case UCALL_ABORT: - TEST_ASSERT(false, "%s at %s:%d", (const char *)uc.args[0], - __FILE__, uc.args[1]); + TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], + __FILE__, uc.args[1]); /* NOT REACHED */ case UCALL_SYNC: break; case UCALL_DONE: - goto done; + goto part1_done; default: - TEST_ASSERT(false, "Unknown ucall 0x%x.", uc.cmd); + TEST_FAIL("Unknown ucall %lu", uc.cmd); } /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && - uc.args[1] == stage, "Unexpected register values vmexit #%lx, got %lx", + uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", stage, (ulong)uc.args[1]); state = vcpu_save_state(vm, VCPU_ID); @@ -147,6 +156,10 @@ int main(int argc, char *argv[]) (ulong) regs2.rdi, (ulong) regs2.rsi); } -done: +part1_done: + _vcpu_run(vm, VCPU_ID); + TEST_ASSERT(run->exit_reason == KVM_EXIT_SHUTDOWN, + "Unexpected successful VMEnter with invalid eVMCS pointer!"); + kvm_vm_free(vm); } diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c b/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c index 443a2b54645b..83323f3d7ca0 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c @@ -66,7 +66,7 @@ static void test_hv_cpuid(struct kvm_cpuid2 *hv_cpuid_entries, TEST_ASSERT((entry->function >= 0x40000000) && (entry->function <= 0x4000000A), - "function %lx is our of supported range", + "function %x is our of supported range", entry->function); TEST_ASSERT(entry->index == 0, @@ -141,8 +141,7 @@ int main(int argc, char *argv[]) rv = kvm_check_cap(KVM_CAP_HYPERV_CPUID); if (!rv) { - fprintf(stderr, - "KVM_CAP_HYPERV_CPUID not supported, skip test\n"); + print_skip("KVM_CAP_HYPERV_CPUID not supported"); exit(KSFT_SKIP); } @@ -160,8 +159,7 @@ int main(int argc, char *argv[]) free(hv_cpuid_entries); if (!kvm_check_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)) { - fprintf(stderr, - "Enlightened VMCS is unsupported, skip related test\n"); + print_skip("Enlightened VMCS is unsupported"); goto vm_free; } diff --git a/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c b/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c index 00bb97d76000..e6480fd5c4bd 100644 --- a/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c +++ b/tools/testing/selftests/kvm/x86_64/mmio_warning_test.c @@ -44,7 +44,7 @@ void *thr(void *arg) struct kvm_run *run = tc->run; res = ioctl(kvmcpu, KVM_RUN, 0); - printf("ret1=%d exit_reason=%d suberror=%d\n", + pr_info("ret1=%d exit_reason=%d suberror=%d\n", res, run->exit_reason, run->internal.suberror); return 0; @@ -93,12 +93,12 @@ int main(void) int warnings_before, warnings_after; if (!is_intel_cpu()) { - printf("Must be run on an Intel CPU, skipping test\n"); + print_skip("Must be run on an Intel CPU"); exit(KSFT_SKIP); } if (vm_is_unrestricted_guest(NULL)) { - printf("Unrestricted guest must be disabled, skipping test\n"); + print_skip("Unrestricted guest must be disabled"); exit(KSFT_SKIP); } diff --git a/tools/testing/selftests/kvm/x86_64/platform_info_test.c b/tools/testing/selftests/kvm/x86_64/platform_info_test.c index f9334bd3cce9..1e89688cbbbf 100644 --- a/tools/testing/selftests/kvm/x86_64/platform_info_test.c +++ b/tools/testing/selftests/kvm/x86_64/platform_info_test.c @@ -58,8 +58,7 @@ static void test_msr_platform_info_enabled(struct kvm_vm *vm) exit_reason_str(run->exit_reason)); get_ucall(vm, VCPU_ID, &uc); TEST_ASSERT(uc.cmd == UCALL_SYNC, - "Received ucall other than UCALL_SYNC: %u\n", - ucall); + "Received ucall other than UCALL_SYNC: %lu\n", uc.cmd); TEST_ASSERT((uc.args[1] & MSR_PLATFORM_INFO_MAX_TURBO_RATIO) == MSR_PLATFORM_INFO_MAX_TURBO_RATIO, "Expected MSR_PLATFORM_INFO to have max turbo ratio mask: %i.", @@ -89,8 +88,7 @@ int main(int argc, char *argv[]) rv = kvm_check_cap(KVM_CAP_MSR_PLATFORM_INFO); if (!rv) { - fprintf(stderr, - "KVM_CAP_MSR_PLATFORM_INFO not supported, skip test\n"); + print_skip("KVM_CAP_MSR_PLATFORM_INFO not supported"); exit(KSFT_SKIP); } diff --git a/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c b/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c new file mode 100644 index 000000000000..c6691cff4e19 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE /* for program_invocation_short_name */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#define VCPU_ID 0 + +/* + * Somewhat arbitrary location and slot, intended to not overlap anything. The + * location and size are specifically 2mb sized/aligned so that the initial + * region corresponds to exactly one large page. + */ +#define MEM_REGION_GPA 0xc0000000 +#define MEM_REGION_SIZE 0x200000 +#define MEM_REGION_SLOT 10 + +static void guest_code(void) +{ + uint64_t val; + + do { + val = READ_ONCE(*((uint64_t *)MEM_REGION_GPA)); + } while (!val); + + if (val != 1) + ucall(UCALL_ABORT, 1, val); + + GUEST_DONE(); +} + +static void *vcpu_worker(void *data) +{ + struct kvm_vm *vm = data; + struct kvm_run *run; + struct ucall uc; + uint64_t cmd; + + /* + * Loop until the guest is done. Re-enter the guest on all MMIO exits, + * which will occur if the guest attempts to access a memslot while it + * is being moved. + */ + run = vcpu_state(vm, VCPU_ID); + do { + vcpu_run(vm, VCPU_ID); + } while (run->exit_reason == KVM_EXIT_MMIO); + + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "Unexpected exit reason = %d", run->exit_reason); + + cmd = get_ucall(vm, VCPU_ID, &uc); + TEST_ASSERT(cmd == UCALL_DONE, "Unexpected val in guest = %lu", uc.args[0]); + return NULL; +} + +static void test_move_memory_region(void) +{ + pthread_t vcpu_thread; + struct kvm_vm *vm; + uint64_t *hva; + uint64_t gpa; + + vm = vm_create_default(VCPU_ID, 0, guest_code); + + vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); + + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP, + MEM_REGION_GPA, MEM_REGION_SLOT, + MEM_REGION_SIZE / getpagesize(), 0); + + /* + * Allocate and map two pages so that the GPA accessed by guest_code() + * stays valid across the memslot move. + */ + gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT); + TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n"); + + virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2, 0); + + /* Ditto for the host mapping so that both pages can be zeroed. */ + hva = addr_gpa2hva(vm, MEM_REGION_GPA); + memset(hva, 0, 2 * 4096); + + pthread_create(&vcpu_thread, NULL, vcpu_worker, vm); + + /* Ensure the guest thread is spun up. */ + usleep(100000); + + /* + * Shift the region's base GPA. The guest should not see "2" as the + * hva->gpa translation is misaligned, i.e. the guest is accessing a + * different host pfn. + */ + vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA - 4096); + WRITE_ONCE(*hva, 2); + + usleep(100000); + + /* + * Note, value in memory needs to be changed *before* restoring the + * memslot, else the guest could race the update and see "2". + */ + WRITE_ONCE(*hva, 1); + + /* Restore the original base, the guest should see "1". */ + vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA); + + pthread_join(vcpu_thread, NULL); + + kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + int i, loops; + + /* Tell stdout not to buffer its content */ + setbuf(stdout, NULL); + + if (argc > 1) + loops = atoi(argv[1]); + else + loops = 10; + + for (i = 0; i < loops; i++) + test_move_memory_region(); + + return 0; +} diff --git a/tools/testing/selftests/kvm/x86_64/smm_test.c b/tools/testing/selftests/kvm/x86_64/smm_test.c index 8c063646f2a0..8230b6bc6b8f 100644 --- a/tools/testing/selftests/kvm/x86_64/smm_test.c +++ b/tools/testing/selftests/kvm/x86_64/smm_test.c @@ -117,7 +117,7 @@ int main(int argc, char *argv[]) vcpu_alloc_vmx(vm, &vmx_pages_gva); vcpu_args_set(vm, VCPU_ID, 1, vmx_pages_gva); } else { - printf("will skip SMM test with VMX enabled\n"); + pr_info("will skip SMM test with VMX enabled\n"); vcpu_args_set(vm, VCPU_ID, 1, 0); } diff --git a/tools/testing/selftests/kvm/x86_64/state_test.c b/tools/testing/selftests/kvm/x86_64/state_test.c index 3ab5ec3da9f4..5b1a016edf55 100644 --- a/tools/testing/selftests/kvm/x86_64/state_test.c +++ b/tools/testing/selftests/kvm/x86_64/state_test.c @@ -139,7 +139,7 @@ int main(int argc, char *argv[]) vcpu_alloc_vmx(vm, &vmx_pages_gva); vcpu_args_set(vm, VCPU_ID, 1, vmx_pages_gva); } else { - printf("will skip nested state checks\n"); + pr_info("will skip nested state checks\n"); vcpu_args_set(vm, VCPU_ID, 1, 0); } @@ -152,20 +152,20 @@ int main(int argc, char *argv[]) switch (get_ucall(vm, VCPU_ID, &uc)) { case UCALL_ABORT: - TEST_ASSERT(false, "%s at %s:%d", (const char *)uc.args[0], - __FILE__, uc.args[1]); + TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], + __FILE__, uc.args[1]); /* NOT REACHED */ case UCALL_SYNC: break; case UCALL_DONE: goto done; default: - TEST_ASSERT(false, "Unknown ucall 0x%x.", uc.cmd); + TEST_FAIL("Unknown ucall %lu", uc.cmd); } /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && - uc.args[1] == stage, "Unexpected register values vmexit #%lx, got %lx", + uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", stage, (ulong)uc.args[1]); state = vcpu_save_state(vm, VCPU_ID); diff --git a/tools/testing/selftests/kvm/x86_64/svm_vmcall_test.c b/tools/testing/selftests/kvm/x86_64/svm_vmcall_test.c index e280f68f6365..0e1adb4e3199 100644 --- a/tools/testing/selftests/kvm/x86_64/svm_vmcall_test.c +++ b/tools/testing/selftests/kvm/x86_64/svm_vmcall_test.c @@ -61,16 +61,14 @@ int main(int argc, char *argv[]) switch (get_ucall(vm, VCPU_ID, &uc)) { case UCALL_ABORT: - TEST_ASSERT(false, "%s", - (const char *)uc.args[0]); + TEST_FAIL("%s", (const char *)uc.args[0]); /* NOT REACHED */ case UCALL_SYNC: break; case UCALL_DONE: goto done; default: - TEST_ASSERT(false, - "Unknown ucall 0x%x.", uc.cmd); + TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd); } } done: diff --git a/tools/testing/selftests/kvm/x86_64/sync_regs_test.c b/tools/testing/selftests/kvm/x86_64/sync_regs_test.c index 5c8224256294..d672f0a473f8 100644 --- a/tools/testing/selftests/kvm/x86_64/sync_regs_test.c +++ b/tools/testing/selftests/kvm/x86_64/sync_regs_test.c @@ -91,11 +91,11 @@ int main(int argc, char *argv[]) cap = kvm_check_cap(KVM_CAP_SYNC_REGS); if ((cap & TEST_SYNC_FIELDS) != TEST_SYNC_FIELDS) { - fprintf(stderr, "KVM_CAP_SYNC_REGS not supported, skipping test\n"); + print_skip("KVM_CAP_SYNC_REGS not supported"); exit(KSFT_SKIP); } if ((cap & INVALID_SYNC_FIELD) != 0) { - fprintf(stderr, "The \"invalid\" field is not invalid, skipping test\n"); + print_skip("The \"invalid\" field is not invalid"); exit(KSFT_SKIP); } diff --git a/tools/testing/selftests/kvm/x86_64/vmx_close_while_nested_test.c b/tools/testing/selftests/kvm/x86_64/vmx_close_while_nested_test.c index 5dfb53546a26..fe40ade06a49 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_close_while_nested_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_close_while_nested_test.c @@ -78,10 +78,10 @@ int main(int argc, char *argv[]) switch (get_ucall(vm, VCPU_ID, &uc)) { case UCALL_ABORT: - TEST_ASSERT(false, "%s", (const char *)uc.args[0]); + TEST_FAIL("%s", (const char *)uc.args[0]); /* NOT REACHED */ default: - TEST_ASSERT(false, "Unknown ucall 0x%x.", uc.cmd); + TEST_FAIL("Unknown ucall %lu", uc.cmd); } } } diff --git a/tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c b/tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c index a223a6401258..e894a638a155 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c @@ -21,7 +21,7 @@ /* The memory slot index to track dirty pages */ #define TEST_MEM_SLOT_INDEX 1 -#define TEST_MEM_SIZE 3 +#define TEST_MEM_PAGES 3 /* L1 guest test virtual memory offset */ #define GUEST_TEST_MEM 0xc0000000 @@ -91,15 +91,14 @@ int main(int argc, char *argv[]) vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, GUEST_TEST_MEM, TEST_MEM_SLOT_INDEX, - TEST_MEM_SIZE, + TEST_MEM_PAGES, KVM_MEM_LOG_DIRTY_PAGES); /* * Add an identity map for GVA range [0xc0000000, 0xc0002000). This * affects both L1 and L2. However... */ - virt_map(vm, GUEST_TEST_MEM, GUEST_TEST_MEM, - TEST_MEM_SIZE * 4096, 0); + virt_map(vm, GUEST_TEST_MEM, GUEST_TEST_MEM, TEST_MEM_PAGES, 0); /* * ... pages in the L2 GPA range [0xc0001000, 0xc0003000) will map to @@ -113,11 +112,11 @@ int main(int argc, char *argv[]) nested_map(vmx, vm, NESTED_TEST_MEM1, GUEST_TEST_MEM, 4096, 0); nested_map(vmx, vm, NESTED_TEST_MEM2, GUEST_TEST_MEM, 4096, 0); - bmap = bitmap_alloc(TEST_MEM_SIZE); + bmap = bitmap_alloc(TEST_MEM_PAGES); host_test_mem = addr_gpa2hva(vm, GUEST_TEST_MEM); while (!done) { - memset(host_test_mem, 0xaa, TEST_MEM_SIZE * 4096); + memset(host_test_mem, 0xaa, TEST_MEM_PAGES * 4096); _vcpu_run(vm, VCPU_ID); TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, "Unexpected exit reason: %u (%s),\n", @@ -126,8 +125,8 @@ int main(int argc, char *argv[]) switch (get_ucall(vm, VCPU_ID, &uc)) { case UCALL_ABORT: - TEST_ASSERT(false, "%s at %s:%d", (const char *)uc.args[0], - __FILE__, uc.args[1]); + TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], + __FILE__, uc.args[1]); /* NOT REACHED */ case UCALL_SYNC: /* @@ -152,7 +151,7 @@ int main(int argc, char *argv[]) done = true; break; default: - TEST_ASSERT(false, "Unknown ucall 0x%x.", uc.cmd); + TEST_FAIL("Unknown ucall %lu", uc.cmd); } } } diff --git a/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c b/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c index 9ef7fab39d48..54cdefdfb49d 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c @@ -212,7 +212,7 @@ void test_vmx_nested_state(struct kvm_vm *vm) test_nested_state(vm, state); vcpu_nested_state_get(vm, VCPU_ID, state); TEST_ASSERT(state->size >= sizeof(*state) && state->size <= state_sz, - "Size must be between %d and %d. The size returned was %d.", + "Size must be between %ld and %d. The size returned was %d.", sizeof(*state), state_sz, state->size); TEST_ASSERT(state->hdr.vmx.vmxon_pa == -1ull, "vmxon_pa must be -1ull."); TEST_ASSERT(state->hdr.vmx.vmcs12_pa == -1ull, "vmcs_pa must be -1ull."); @@ -228,7 +228,7 @@ int main(int argc, char *argv[]) have_evmcs = kvm_check_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS); if (!kvm_check_cap(KVM_CAP_NESTED_STATE)) { - printf("KVM_CAP_NESTED_STATE not available, skipping test\n"); + print_skip("KVM_CAP_NESTED_STATE not available"); exit(KSFT_SKIP); } diff --git a/tools/testing/selftests/kvm/x86_64/vmx_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86_64/vmx_tsc_adjust_test.c index 69e482a95c47..fbe8417cbc2c 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_tsc_adjust_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_tsc_adjust_test.c @@ -121,8 +121,8 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) static void report(int64_t val) { - printf("IA32_TSC_ADJUST is %ld (%lld * TSC_ADJUST_VALUE + %lld).\n", - val, val / TSC_ADJUST_VALUE, val % TSC_ADJUST_VALUE); + pr_info("IA32_TSC_ADJUST is %ld (%lld * TSC_ADJUST_VALUE + %lld).\n", + val, val / TSC_ADJUST_VALUE, val % TSC_ADJUST_VALUE); } int main(int argc, char *argv[]) @@ -150,7 +150,7 @@ int main(int argc, char *argv[]) switch (get_ucall(vm, VCPU_ID, &uc)) { case UCALL_ABORT: - TEST_ASSERT(false, "%s", (const char *)uc.args[0]); + TEST_FAIL("%s", (const char *)uc.args[0]); /* NOT REACHED */ case UCALL_SYNC: report(uc.args[1]); @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) case UCALL_DONE: goto done; default: - TEST_ASSERT(false, "Unknown ucall 0x%x.", uc.cmd); + TEST_FAIL("Unknown ucall %lu", uc.cmd); } } diff --git a/tools/testing/selftests/kvm/x86_64/xss_msr_test.c b/tools/testing/selftests/kvm/x86_64/xss_msr_test.c index 851ea81b9d9f..3529376747c2 100644 --- a/tools/testing/selftests/kvm/x86_64/xss_msr_test.c +++ b/tools/testing/selftests/kvm/x86_64/xss_msr_test.c @@ -51,7 +51,7 @@ int main(int argc, char *argv[]) xss_supported = entry && !!(entry->eax & X86_FEATURE_XSAVES); } if (!xss_supported) { - printf("IA32_XSS is not supported by the vCPU.\n"); + print_skip("IA32_XSS is not supported by the vCPU"); exit(KSFT_SKIP); } diff --git a/tools/testing/selftests/vm/.gitignore b/tools/testing/selftests/vm/.gitignore index 31b3c98b6d34..d3bed9407773 100644 --- a/tools/testing/selftests/vm/.gitignore +++ b/tools/testing/selftests/vm/.gitignore @@ -14,3 +14,4 @@ virtual_address_range gup_benchmark va_128TBswitch map_fixed_noreplace +write_to_hugetlbfs diff --git a/tools/testing/selftests/vm/Makefile b/tools/testing/selftests/vm/Makefile index 7f9a8a8c31da..d31db052dff6 100644 --- a/tools/testing/selftests/vm/Makefile +++ b/tools/testing/selftests/vm/Makefile @@ -14,6 +14,7 @@ TEST_GEN_FILES += map_fixed_noreplace TEST_GEN_FILES += map_populate TEST_GEN_FILES += mlock-random-test TEST_GEN_FILES += mlock2-tests +TEST_GEN_FILES += mremap_dontunmap TEST_GEN_FILES += on-fault-limit TEST_GEN_FILES += thuge-gen TEST_GEN_FILES += transhuge-stress @@ -22,6 +23,7 @@ TEST_GEN_FILES += userfaultfd ifneq (,$(filter $(ARCH),arm64 ia64 mips64 parisc64 ppc64 riscv64 s390x sh64 sparc64 x86_64)) TEST_GEN_FILES += va_128TBswitch TEST_GEN_FILES += virtual_address_range +TEST_GEN_FILES += write_to_hugetlbfs endif TEST_PROGS := run_vmtests diff --git a/tools/testing/selftests/vm/charge_reserved_hugetlb.sh b/tools/testing/selftests/vm/charge_reserved_hugetlb.sh new file mode 100644 index 000000000000..18d33684faad --- /dev/null +++ b/tools/testing/selftests/vm/charge_reserved_hugetlb.sh @@ -0,0 +1,575 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +set -e + +if [[ $(id -u) -ne 0 ]]; then + echo "This test must be run as root. Skipping..." + exit 0 +fi + +fault_limit_file=limit_in_bytes +reservation_limit_file=rsvd.limit_in_bytes +fault_usage_file=usage_in_bytes +reservation_usage_file=rsvd.usage_in_bytes + +if [[ "$1" == "-cgroup-v2" ]]; then + cgroup2=1 + fault_limit_file=max + reservation_limit_file=rsvd.max + fault_usage_file=current + reservation_usage_file=rsvd.current +fi + +cgroup_path=/dev/cgroup/memory +if [[ ! -e $cgroup_path ]]; then + mkdir -p $cgroup_path + if [[ $cgroup2 ]]; then + mount -t cgroup2 none $cgroup_path + else + mount -t cgroup memory,hugetlb $cgroup_path + fi +fi + +if [[ $cgroup2 ]]; then + echo "+hugetlb" >/dev/cgroup/memory/cgroup.subtree_control +fi + +function cleanup() { + if [[ $cgroup2 ]]; then + echo $$ >$cgroup_path/cgroup.procs + else + echo $$ >$cgroup_path/tasks + fi + + if [[ -e /mnt/huge ]]; then + rm -rf /mnt/huge/* + umount /mnt/huge || echo error + rmdir /mnt/huge + fi + if [[ -e $cgroup_path/hugetlb_cgroup_test ]]; then + rmdir $cgroup_path/hugetlb_cgroup_test + fi + if [[ -e $cgroup_path/hugetlb_cgroup_test1 ]]; then + rmdir $cgroup_path/hugetlb_cgroup_test1 + fi + if [[ -e $cgroup_path/hugetlb_cgroup_test2 ]]; then + rmdir $cgroup_path/hugetlb_cgroup_test2 + fi + echo 0 >/proc/sys/vm/nr_hugepages + echo CLEANUP DONE +} + +function expect_equal() { + local expected="$1" + local actual="$2" + local error="$3" + + if [[ "$expected" != "$actual" ]]; then + echo "expected ($expected) != actual ($actual): $3" + cleanup + exit 1 + fi +} + +function get_machine_hugepage_size() { + hpz=$(grep -i hugepagesize /proc/meminfo) + kb=${hpz:14:-3} + mb=$(($kb / 1024)) + echo $mb +} + +MB=$(get_machine_hugepage_size) + +function setup_cgroup() { + local name="$1" + local cgroup_limit="$2" + local reservation_limit="$3" + + mkdir $cgroup_path/$name + + echo writing cgroup limit: "$cgroup_limit" + echo "$cgroup_limit" >$cgroup_path/$name/hugetlb.${MB}MB.$fault_limit_file + + echo writing reseravation limit: "$reservation_limit" + echo "$reservation_limit" > \ + $cgroup_path/$name/hugetlb.${MB}MB.$reservation_limit_file + + if [ -e "$cgroup_path/$name/cpuset.cpus" ]; then + echo 0 >$cgroup_path/$name/cpuset.cpus + fi + if [ -e "$cgroup_path/$name/cpuset.mems" ]; then + echo 0 >$cgroup_path/$name/cpuset.mems + fi +} + +function wait_for_hugetlb_memory_to_get_depleted() { + local cgroup="$1" + local path="/dev/cgroup/memory/$cgroup/hugetlb.${MB}MB.$reservation_usage_file" + # Wait for hugetlbfs memory to get depleted. + while [ $(cat $path) != 0 ]; do + echo Waiting for hugetlb memory to get depleted. + cat $path + sleep 0.5 + done +} + +function wait_for_hugetlb_memory_to_get_reserved() { + local cgroup="$1" + local size="$2" + + local path="/dev/cgroup/memory/$cgroup/hugetlb.${MB}MB.$reservation_usage_file" + # Wait for hugetlbfs memory to get written. + while [ $(cat $path) != $size ]; do + echo Waiting for hugetlb memory reservation to reach size $size. + cat $path + sleep 0.5 + done +} + +function wait_for_hugetlb_memory_to_get_written() { + local cgroup="$1" + local size="$2" + + local path="/dev/cgroup/memory/$cgroup/hugetlb.${MB}MB.$fault_usage_file" + # Wait for hugetlbfs memory to get written. + while [ $(cat $path) != $size ]; do + echo Waiting for hugetlb memory to reach size $size. + cat $path + sleep 0.5 + done +} + +function write_hugetlbfs_and_get_usage() { + local cgroup="$1" + local size="$2" + local populate="$3" + local write="$4" + local path="$5" + local method="$6" + local private="$7" + local expect_failure="$8" + local reserve="$9" + + # Function return values. + reservation_failed=0 + oom_killed=0 + hugetlb_difference=0 + reserved_difference=0 + + local hugetlb_usage=$cgroup_path/$cgroup/hugetlb.${MB}MB.$fault_usage_file + local reserved_usage=$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file + + local hugetlb_before=$(cat $hugetlb_usage) + local reserved_before=$(cat $reserved_usage) + + echo + echo Starting: + echo hugetlb_usage="$hugetlb_before" + echo reserved_usage="$reserved_before" + echo expect_failure is "$expect_failure" + + output=$(mktemp) + set +e + if [[ "$method" == "1" ]] || [[ "$method" == 2 ]] || + [[ "$private" == "-r" ]] && [[ "$expect_failure" != 1 ]]; then + + bash write_hugetlb_memory.sh "$size" "$populate" "$write" \ + "$cgroup" "$path" "$method" "$private" "-l" "$reserve" 2>&1 | tee $output & + + local write_result=$? + local write_pid=$! + + until grep -q -i "DONE" $output; do + echo waiting for DONE signal. + if ! ps $write_pid > /dev/null + then + echo "FAIL: The write died" + cleanup + exit 1 + fi + sleep 0.5 + done + + echo ================= write_hugetlb_memory.sh output is: + cat $output + echo ================= end output. + + if [[ "$populate" == "-o" ]] || [[ "$write" == "-w" ]]; then + wait_for_hugetlb_memory_to_get_written "$cgroup" "$size" + elif [[ "$reserve" != "-n" ]]; then + wait_for_hugetlb_memory_to_get_reserved "$cgroup" "$size" + else + # This case doesn't produce visible effects, but we still have + # to wait for the async process to start and execute... + sleep 0.5 + fi + + echo write_result is $write_result + else + bash write_hugetlb_memory.sh "$size" "$populate" "$write" \ + "$cgroup" "$path" "$method" "$private" "$reserve" + local write_result=$? + + if [[ "$reserve" != "-n" ]]; then + wait_for_hugetlb_memory_to_get_reserved "$cgroup" "$size" + fi + fi + set -e + + if [[ "$write_result" == 1 ]]; then + reservation_failed=1 + fi + + # On linus/master, the above process gets SIGBUS'd on oomkill, with + # return code 135. On earlier kernels, it gets actual oomkill, with return + # code 137, so just check for both conditions in case we're testing + # against an earlier kernel. + if [[ "$write_result" == 135 ]] || [[ "$write_result" == 137 ]]; then + oom_killed=1 + fi + + local hugetlb_after=$(cat $hugetlb_usage) + local reserved_after=$(cat $reserved_usage) + + echo After write: + echo hugetlb_usage="$hugetlb_after" + echo reserved_usage="$reserved_after" + + hugetlb_difference=$(($hugetlb_after - $hugetlb_before)) + reserved_difference=$(($reserved_after - $reserved_before)) +} + +function cleanup_hugetlb_memory() { + set +e + local cgroup="$1" + if [[ "$(pgrep -f write_to_hugetlbfs)" != "" ]]; then + echo killing write_to_hugetlbfs + killall -2 write_to_hugetlbfs + wait_for_hugetlb_memory_to_get_depleted $cgroup + fi + set -e + + if [[ -e /mnt/huge ]]; then + rm -rf /mnt/huge/* + umount /mnt/huge + rmdir /mnt/huge + fi +} + +function run_test() { + local size=$(($1 * ${MB} * 1024 * 1024)) + local populate="$2" + local write="$3" + local cgroup_limit=$(($4 * ${MB} * 1024 * 1024)) + local reservation_limit=$(($5 * ${MB} * 1024 * 1024)) + local nr_hugepages="$6" + local method="$7" + local private="$8" + local expect_failure="$9" + local reserve="${10}" + + # Function return values. + hugetlb_difference=0 + reserved_difference=0 + reservation_failed=0 + oom_killed=0 + + echo nr hugepages = "$nr_hugepages" + echo "$nr_hugepages" >/proc/sys/vm/nr_hugepages + + setup_cgroup "hugetlb_cgroup_test" "$cgroup_limit" "$reservation_limit" + + mkdir -p /mnt/huge + mount -t hugetlbfs -o pagesize=${MB}M,size=256M none /mnt/huge + + write_hugetlbfs_and_get_usage "hugetlb_cgroup_test" "$size" "$populate" \ + "$write" "/mnt/huge/test" "$method" "$private" "$expect_failure" \ + "$reserve" + + cleanup_hugetlb_memory "hugetlb_cgroup_test" + + local final_hugetlb=$(cat $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB}MB.$fault_usage_file) + local final_reservation=$(cat $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB}MB.$reservation_usage_file) + + echo $hugetlb_difference + echo $reserved_difference + expect_equal "0" "$final_hugetlb" "final hugetlb is not zero" + expect_equal "0" "$final_reservation" "final reservation is not zero" +} + +function run_multiple_cgroup_test() { + local size1="$1" + local populate1="$2" + local write1="$3" + local cgroup_limit1="$4" + local reservation_limit1="$5" + + local size2="$6" + local populate2="$7" + local write2="$8" + local cgroup_limit2="$9" + local reservation_limit2="${10}" + + local nr_hugepages="${11}" + local method="${12}" + local private="${13}" + local expect_failure="${14}" + local reserve="${15}" + + # Function return values. + hugetlb_difference1=0 + reserved_difference1=0 + reservation_failed1=0 + oom_killed1=0 + + hugetlb_difference2=0 + reserved_difference2=0 + reservation_failed2=0 + oom_killed2=0 + + echo nr hugepages = "$nr_hugepages" + echo "$nr_hugepages" >/proc/sys/vm/nr_hugepages + + setup_cgroup "hugetlb_cgroup_test1" "$cgroup_limit1" "$reservation_limit1" + setup_cgroup "hugetlb_cgroup_test2" "$cgroup_limit2" "$reservation_limit2" + + mkdir -p /mnt/huge + mount -t hugetlbfs -o pagesize=${MB}M,size=256M none /mnt/huge + + write_hugetlbfs_and_get_usage "hugetlb_cgroup_test1" "$size1" \ + "$populate1" "$write1" "/mnt/huge/test1" "$method" "$private" \ + "$expect_failure" "$reserve" + + hugetlb_difference1=$hugetlb_difference + reserved_difference1=$reserved_difference + reservation_failed1=$reservation_failed + oom_killed1=$oom_killed + + local cgroup1_hugetlb_usage=$cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB}MB.$fault_usage_file + local cgroup1_reservation_usage=$cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB}MB.$reservation_usage_file + local cgroup2_hugetlb_usage=$cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB}MB.$fault_usage_file + local cgroup2_reservation_usage=$cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB}MB.$reservation_usage_file + + local usage_before_second_write=$(cat $cgroup1_hugetlb_usage) + local reservation_usage_before_second_write=$(cat $cgroup1_reservation_usage) + + write_hugetlbfs_and_get_usage "hugetlb_cgroup_test2" "$size2" \ + "$populate2" "$write2" "/mnt/huge/test2" "$method" "$private" \ + "$expect_failure" "$reserve" + + hugetlb_difference2=$hugetlb_difference + reserved_difference2=$reserved_difference + reservation_failed2=$reservation_failed + oom_killed2=$oom_killed + + expect_equal "$usage_before_second_write" \ + "$(cat $cgroup1_hugetlb_usage)" "Usage changed." + expect_equal "$reservation_usage_before_second_write" \ + "$(cat $cgroup1_reservation_usage)" "Reservation usage changed." + + cleanup_hugetlb_memory + + local final_hugetlb=$(cat $cgroup1_hugetlb_usage) + local final_reservation=$(cat $cgroup1_reservation_usage) + + expect_equal "0" "$final_hugetlb" \ + "hugetlbt_cgroup_test1 final hugetlb is not zero" + expect_equal "0" "$final_reservation" \ + "hugetlbt_cgroup_test1 final reservation is not zero" + + local final_hugetlb=$(cat $cgroup2_hugetlb_usage) + local final_reservation=$(cat $cgroup2_reservation_usage) + + expect_equal "0" "$final_hugetlb" \ + "hugetlb_cgroup_test2 final hugetlb is not zero" + expect_equal "0" "$final_reservation" \ + "hugetlb_cgroup_test2 final reservation is not zero" +} + +cleanup + +for populate in "" "-o"; do + for method in 0 1 2; do + for private in "" "-r"; do + for reserve in "" "-n"; do + + # Skip mmap(MAP_HUGETLB | MAP_SHARED). Doesn't seem to be supported. + if [[ "$method" == 1 ]] && [[ "$private" == "" ]]; then + continue + fi + + # Skip populated shmem tests. Doesn't seem to be supported. + if [[ "$method" == 2"" ]] && [[ "$populate" == "-o" ]]; then + continue + fi + + if [[ "$method" == 2"" ]] && [[ "$reserve" == "-n" ]]; then + continue + fi + + cleanup + echo + echo + echo + echo Test normal case. + echo private=$private, populate=$populate, method=$method, reserve=$reserve + run_test 5 "$populate" "" 10 10 10 "$method" "$private" "0" "$reserve" + + echo Memory charged to hugtlb=$hugetlb_difference + echo Memory charged to reservation=$reserved_difference + + if [[ "$populate" == "-o" ]]; then + expect_equal "$((5 * $MB * 1024 * 1024))" "$hugetlb_difference" \ + "Reserved memory charged to hugetlb cgroup." + else + expect_equal "0" "$hugetlb_difference" \ + "Reserved memory charged to hugetlb cgroup." + fi + + if [[ "$reserve" != "-n" ]] || [[ "$populate" == "-o" ]]; then + expect_equal "$((5 * $MB * 1024 * 1024))" "$reserved_difference" \ + "Reserved memory not charged to reservation usage." + else + expect_equal "0" "$reserved_difference" \ + "Reserved memory not charged to reservation usage." + fi + + echo 'PASS' + + cleanup + echo + echo + echo + echo Test normal case with write. + echo private=$private, populate=$populate, method=$method, reserve=$reserve + run_test 5 "$populate" '-w' 5 5 10 "$method" "$private" "0" "$reserve" + + echo Memory charged to hugtlb=$hugetlb_difference + echo Memory charged to reservation=$reserved_difference + + expect_equal "$((5 * $MB * 1024 * 1024))" "$hugetlb_difference" \ + "Reserved memory charged to hugetlb cgroup." + + expect_equal "$((5 * $MB * 1024 * 1024))" "$reserved_difference" \ + "Reserved memory not charged to reservation usage." + + echo 'PASS' + + cleanup + continue + echo + echo + echo + echo Test more than reservation case. + echo private=$private, populate=$populate, method=$method, reserve=$reserve + + if [ "$reserve" != "-n" ]; then + run_test "5" "$populate" '' "10" "2" "10" "$method" "$private" "1" \ + "$reserve" + + expect_equal "1" "$reservation_failed" "Reservation succeeded." + fi + + echo 'PASS' + + cleanup + + echo + echo + echo + echo Test more than cgroup limit case. + echo private=$private, populate=$populate, method=$method, reserve=$reserve + + # Not sure if shm memory can be cleaned up when the process gets sigbus'd. + if [[ "$method" != 2 ]]; then + run_test 5 "$populate" "-w" 2 10 10 "$method" "$private" "1" "$reserve" + + expect_equal "1" "$oom_killed" "Not oom killed." + fi + echo 'PASS' + + cleanup + + echo + echo + echo + echo Test normal case, multiple cgroups. + echo private=$private, populate=$populate, method=$method, reserve=$reserve + run_multiple_cgroup_test "3" "$populate" "" "10" "10" "5" \ + "$populate" "" "10" "10" "10" \ + "$method" "$private" "0" "$reserve" + + echo Memory charged to hugtlb1=$hugetlb_difference1 + echo Memory charged to reservation1=$reserved_difference1 + echo Memory charged to hugtlb2=$hugetlb_difference2 + echo Memory charged to reservation2=$reserved_difference2 + + if [[ "$reserve" != "-n" ]] || [[ "$populate" == "-o" ]]; then + expect_equal "3" "$reserved_difference1" \ + "Incorrect reservations charged to cgroup 1." + + expect_equal "5" "$reserved_difference2" \ + "Incorrect reservation charged to cgroup 2." + + else + expect_equal "0" "$reserved_difference1" \ + "Incorrect reservations charged to cgroup 1." + + expect_equal "0" "$reserved_difference2" \ + "Incorrect reservation charged to cgroup 2." + fi + + if [[ "$populate" == "-o" ]]; then + expect_equal "3" "$hugetlb_difference1" \ + "Incorrect hugetlb charged to cgroup 1." + + expect_equal "5" "$hugetlb_difference2" \ + "Incorrect hugetlb charged to cgroup 2." + + else + expect_equal "0" "$hugetlb_difference1" \ + "Incorrect hugetlb charged to cgroup 1." + + expect_equal "0" "$hugetlb_difference2" \ + "Incorrect hugetlb charged to cgroup 2." + fi + echo 'PASS' + + cleanup + echo + echo + echo + echo Test normal case with write, multiple cgroups. + echo private=$private, populate=$populate, method=$method, reserve=$reserve + run_multiple_cgroup_test "3" "$populate" "-w" "10" "10" "5" \ + "$populate" "-w" "10" "10" "10" \ + "$method" "$private" "0" "$reserve" + + echo Memory charged to hugtlb1=$hugetlb_difference1 + echo Memory charged to reservation1=$reserved_difference1 + echo Memory charged to hugtlb2=$hugetlb_difference2 + echo Memory charged to reservation2=$reserved_difference2 + + expect_equal "3" "$hugetlb_difference1" \ + "Incorrect hugetlb charged to cgroup 1." + + expect_equal "3" "$reserved_difference1" \ + "Incorrect reservation charged to cgroup 1." + + expect_equal "5" "$hugetlb_difference2" \ + "Incorrect hugetlb charged to cgroup 2." + + expect_equal "5" "$reserved_difference2" \ + "Incorrected reservation charged to cgroup 2." + echo 'PASS' + + cleanup + + done # reserve + done # private + done # populate +done # method + +umount $cgroup_path +rmdir $cgroup_path diff --git a/tools/testing/selftests/vm/gup_benchmark.c b/tools/testing/selftests/vm/gup_benchmark.c index 389327e9b30a..43b4dfe161a2 100644 --- a/tools/testing/selftests/vm/gup_benchmark.c +++ b/tools/testing/selftests/vm/gup_benchmark.c @@ -18,6 +18,10 @@ #define GUP_LONGTERM_BENCHMARK _IOWR('g', 2, struct gup_benchmark) #define GUP_BENCHMARK _IOWR('g', 3, struct gup_benchmark) +/* Similar to above, but use FOLL_PIN instead of FOLL_GET. */ +#define PIN_FAST_BENCHMARK _IOWR('g', 4, struct gup_benchmark) +#define PIN_BENCHMARK _IOWR('g', 5, struct gup_benchmark) + /* Just the flags we need, copied from mm.h: */ #define FOLL_WRITE 0x01 /* check pte is writable */ @@ -40,8 +44,14 @@ int main(int argc, char **argv) char *file = "/dev/zero"; char *p; - while ((opt = getopt(argc, argv, "m:r:n:f:tTLUwSH")) != -1) { + while ((opt = getopt(argc, argv, "m:r:n:f:abtTLUuwSH")) != -1) { switch (opt) { + case 'a': + cmd = PIN_FAST_BENCHMARK; + break; + case 'b': + cmd = PIN_BENCHMARK; + break; case 'm': size = atoi(optarg) * MB; break; @@ -63,6 +73,9 @@ int main(int argc, char **argv) case 'U': cmd = GUP_BENCHMARK; break; + case 'u': + cmd = GUP_FAST_BENCHMARK; + break; case 'w': write = 1; break; diff --git a/tools/testing/selftests/vm/hugetlb_reparenting_test.sh b/tools/testing/selftests/vm/hugetlb_reparenting_test.sh new file mode 100644 index 000000000000..d11d1febccc3 --- /dev/null +++ b/tools/testing/selftests/vm/hugetlb_reparenting_test.sh @@ -0,0 +1,244 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +set -e + +if [[ $(id -u) -ne 0 ]]; then + echo "This test must be run as root. Skipping..." + exit 0 +fi + +usage_file=usage_in_bytes + +if [[ "$1" == "-cgroup-v2" ]]; then + cgroup2=1 + usage_file=current +fi + +CGROUP_ROOT='/dev/cgroup/memory' +MNT='/mnt/huge/' + +if [[ ! -e $CGROUP_ROOT ]]; then + mkdir -p $CGROUP_ROOT + if [[ $cgroup2 ]]; then + mount -t cgroup2 none $CGROUP_ROOT + sleep 1 + echo "+hugetlb +memory" >$CGROUP_ROOT/cgroup.subtree_control + else + mount -t cgroup memory,hugetlb $CGROUP_ROOT + fi +fi + +function get_machine_hugepage_size() { + hpz=$(grep -i hugepagesize /proc/meminfo) + kb=${hpz:14:-3} + mb=$(($kb / 1024)) + echo $mb +} + +MB=$(get_machine_hugepage_size) + +function cleanup() { + echo cleanup + set +e + rm -rf "$MNT"/* 2>/dev/null + umount "$MNT" 2>/dev/null + rmdir "$MNT" 2>/dev/null + rmdir "$CGROUP_ROOT"/a/b 2>/dev/null + rmdir "$CGROUP_ROOT"/a 2>/dev/null + rmdir "$CGROUP_ROOT"/test1 2>/dev/null + echo 0 >/proc/sys/vm/nr_hugepages + set -e +} + +function assert_state() { + local expected_a="$1" + local expected_a_hugetlb="$2" + local expected_b="" + local expected_b_hugetlb="" + + if [ ! -z ${3:-} ] && [ ! -z ${4:-} ]; then + expected_b="$3" + expected_b_hugetlb="$4" + fi + local tolerance=$((5 * 1024 * 1024)) + + local actual_a + actual_a="$(cat "$CGROUP_ROOT"/a/memory.$usage_file)" + if [[ $actual_a -lt $(($expected_a - $tolerance)) ]] || + [[ $actual_a -gt $(($expected_a + $tolerance)) ]]; then + echo actual a = $((${actual_a%% *} / 1024 / 1024)) MB + echo expected a = $((${expected_a%% *} / 1024 / 1024)) MB + echo fail + + cleanup + exit 1 + fi + + local actual_a_hugetlb + actual_a_hugetlb="$(cat "$CGROUP_ROOT"/a/hugetlb.${MB}MB.$usage_file)" + if [[ $actual_a_hugetlb -lt $(($expected_a_hugetlb - $tolerance)) ]] || + [[ $actual_a_hugetlb -gt $(($expected_a_hugetlb + $tolerance)) ]]; then + echo actual a hugetlb = $((${actual_a_hugetlb%% *} / 1024 / 1024)) MB + echo expected a hugetlb = $((${expected_a_hugetlb%% *} / 1024 / 1024)) MB + echo fail + + cleanup + exit 1 + fi + + if [[ -z "$expected_b" || -z "$expected_b_hugetlb" ]]; then + return + fi + + local actual_b + actual_b="$(cat "$CGROUP_ROOT"/a/b/memory.$usage_file)" + if [[ $actual_b -lt $(($expected_b - $tolerance)) ]] || + [[ $actual_b -gt $(($expected_b + $tolerance)) ]]; then + echo actual b = $((${actual_b%% *} / 1024 / 1024)) MB + echo expected b = $((${expected_b%% *} / 1024 / 1024)) MB + echo fail + + cleanup + exit 1 + fi + + local actual_b_hugetlb + actual_b_hugetlb="$(cat "$CGROUP_ROOT"/a/b/hugetlb.${MB}MB.$usage_file)" + if [[ $actual_b_hugetlb -lt $(($expected_b_hugetlb - $tolerance)) ]] || + [[ $actual_b_hugetlb -gt $(($expected_b_hugetlb + $tolerance)) ]]; then + echo actual b hugetlb = $((${actual_b_hugetlb%% *} / 1024 / 1024)) MB + echo expected b hugetlb = $((${expected_b_hugetlb%% *} / 1024 / 1024)) MB + echo fail + + cleanup + exit 1 + fi +} + +function setup() { + echo 100 >/proc/sys/vm/nr_hugepages + mkdir "$CGROUP_ROOT"/a + sleep 1 + if [[ $cgroup2 ]]; then + echo "+hugetlb +memory" >$CGROUP_ROOT/a/cgroup.subtree_control + else + echo 0 >$CGROUP_ROOT/a/cpuset.mems + echo 0 >$CGROUP_ROOT/a/cpuset.cpus + fi + + mkdir "$CGROUP_ROOT"/a/b + + if [[ ! $cgroup2 ]]; then + echo 0 >$CGROUP_ROOT/a/b/cpuset.mems + echo 0 >$CGROUP_ROOT/a/b/cpuset.cpus + fi + + mkdir -p "$MNT" + mount -t hugetlbfs none "$MNT" +} + +write_hugetlbfs() { + local cgroup="$1" + local path="$2" + local size="$3" + + if [[ $cgroup2 ]]; then + echo $$ >$CGROUP_ROOT/$cgroup/cgroup.procs + else + echo 0 >$CGROUP_ROOT/$cgroup/cpuset.mems + echo 0 >$CGROUP_ROOT/$cgroup/cpuset.cpus + echo $$ >"$CGROUP_ROOT/$cgroup/tasks" + fi + ./write_to_hugetlbfs -p "$path" -s "$size" -m 0 -o + if [[ $cgroup2 ]]; then + echo $$ >$CGROUP_ROOT/cgroup.procs + else + echo $$ >"$CGROUP_ROOT/tasks" + fi + echo +} + +set -e + +size=$((${MB} * 1024 * 1024 * 25)) # 50MB = 25 * 2MB hugepages. + +cleanup + +echo +echo +echo Test charge, rmdir, uncharge +setup +echo mkdir +mkdir $CGROUP_ROOT/test1 + +echo write +write_hugetlbfs test1 "$MNT"/test $size + +echo rmdir +rmdir $CGROUP_ROOT/test1 +mkdir $CGROUP_ROOT/test1 + +echo uncharge +rm -rf /mnt/huge/* + +cleanup + +echo done +echo +echo +if [[ ! $cgroup2 ]]; then + echo "Test parent and child hugetlb usage" + setup + + echo write + write_hugetlbfs a "$MNT"/test $size + + echo Assert memory charged correctly for parent use. + assert_state 0 $size 0 0 + + write_hugetlbfs a/b "$MNT"/test2 $size + + echo Assert memory charged correctly for child use. + assert_state 0 $(($size * 2)) 0 $size + + rmdir "$CGROUP_ROOT"/a/b + sleep 5 + echo Assert memory reparent correctly. + assert_state 0 $(($size * 2)) + + rm -rf "$MNT"/* + umount "$MNT" + echo Assert memory uncharged correctly. + assert_state 0 0 + + cleanup +fi + +echo +echo +echo "Test child only hugetlb usage" +echo setup +setup + +echo write +write_hugetlbfs a/b "$MNT"/test2 $size + +echo Assert memory charged correctly for child only use. +assert_state 0 $(($size)) 0 $size + +rmdir "$CGROUP_ROOT"/a/b +echo Assert memory reparent correctly. +assert_state 0 $size + +rm -rf "$MNT"/* +umount "$MNT" +echo Assert memory uncharged correctly. +assert_state 0 0 + +cleanup + +echo ALL PASS + +umount $CGROUP_ROOT +rm -rf $CGROUP_ROOT diff --git a/tools/testing/selftests/vm/map_hugetlb.c b/tools/testing/selftests/vm/map_hugetlb.c index 5a2d7b8efc40..6af951900aa3 100644 --- a/tools/testing/selftests/vm/map_hugetlb.c +++ b/tools/testing/selftests/vm/map_hugetlb.c @@ -45,20 +45,20 @@ static void check_bytes(char *addr) printf("First hex is %x\n", *((unsigned int *)addr)); } -static void write_bytes(char *addr) +static void write_bytes(char *addr, size_t length) { unsigned long i; - for (i = 0; i < LENGTH; i++) + for (i = 0; i < length; i++) *(addr + i) = (char)i; } -static int read_bytes(char *addr) +static int read_bytes(char *addr, size_t length) { unsigned long i; check_bytes(addr); - for (i = 0; i < LENGTH; i++) + for (i = 0; i < length; i++) if (*(addr + i) != (char)i) { printf("Mismatch at %lu\n", i); return 1; @@ -96,11 +96,11 @@ int main(int argc, char **argv) printf("Returned address is %p\n", addr); check_bytes(addr); - write_bytes(addr); - ret = read_bytes(addr); + write_bytes(addr, length); + ret = read_bytes(addr, length); /* munmap() length of MAP_HUGETLB memory must be hugepage aligned */ - if (munmap(addr, LENGTH)) { + if (munmap(addr, length)) { perror("munmap"); exit(1); } diff --git a/tools/testing/selftests/vm/mlock2-tests.c b/tools/testing/selftests/vm/mlock2-tests.c index 637b6d0ac0d0..11b2301f3aa3 100644 --- a/tools/testing/selftests/vm/mlock2-tests.c +++ b/tools/testing/selftests/vm/mlock2-tests.c @@ -67,59 +67,6 @@ static int get_vm_area(unsigned long addr, struct vm_boundaries *area) return ret; } -static uint64_t get_pageflags(unsigned long addr) -{ - FILE *file; - uint64_t pfn; - unsigned long offset; - - file = fopen("/proc/self/pagemap", "r"); - if (!file) { - perror("fopen pagemap"); - _exit(1); - } - - offset = addr / getpagesize() * sizeof(pfn); - - if (fseek(file, offset, SEEK_SET)) { - perror("fseek pagemap"); - _exit(1); - } - - if (fread(&pfn, sizeof(pfn), 1, file) != 1) { - perror("fread pagemap"); - _exit(1); - } - - fclose(file); - return pfn; -} - -static uint64_t get_kpageflags(unsigned long pfn) -{ - uint64_t flags; - FILE *file; - - file = fopen("/proc/kpageflags", "r"); - if (!file) { - perror("fopen kpageflags"); - _exit(1); - } - - if (fseek(file, pfn * sizeof(flags), SEEK_SET)) { - perror("fseek kpageflags"); - _exit(1); - } - - if (fread(&flags, sizeof(flags), 1, file) != 1) { - perror("fread kpageflags"); - _exit(1); - } - - fclose(file); - return flags; -} - #define VMFLAGS "VmFlags:" static bool is_vmflag_set(unsigned long addr, const char *vmflag) @@ -159,19 +106,13 @@ static bool is_vmflag_set(unsigned long addr, const char *vmflag) #define RSS "Rss:" #define LOCKED "lo" -static bool is_vma_lock_on_fault(unsigned long addr) +static unsigned long get_value_for_name(unsigned long addr, const char *name) { - bool ret = false; - bool locked; - FILE *smaps = NULL; - unsigned long vma_size, vma_rss; char *line = NULL; - char *value; size_t size = 0; - - locked = is_vmflag_set(addr, LOCKED); - if (!locked) - goto out; + char *value_ptr; + FILE *smaps = NULL; + unsigned long value = -1UL; smaps = seek_to_smaps_entry(addr); if (!smaps) { @@ -180,112 +121,70 @@ static bool is_vma_lock_on_fault(unsigned long addr) } while (getline(&line, &size, smaps) > 0) { - if (!strstr(line, SIZE)) { + if (!strstr(line, name)) { free(line); line = NULL; size = 0; continue; } - value = line + strlen(SIZE); - if (sscanf(value, "%lu kB", &vma_size) < 1) { + value_ptr = line + strlen(name); + if (sscanf(value_ptr, "%lu kB", &value) < 1) { printf("Unable to parse smaps entry for Size\n"); goto out; } break; } - while (getline(&line, &size, smaps) > 0) { - if (!strstr(line, RSS)) { - free(line); - line = NULL; - size = 0; - continue; - } - - value = line + strlen(RSS); - if (sscanf(value, "%lu kB", &vma_rss) < 1) { - printf("Unable to parse smaps entry for Rss\n"); - goto out; - } - break; - } - - ret = locked && (vma_rss < vma_size); out: - free(line); if (smaps) fclose(smaps); - return ret; + free(line); + return value; +} + +static bool is_vma_lock_on_fault(unsigned long addr) +{ + bool locked; + unsigned long vma_size, vma_rss; + + locked = is_vmflag_set(addr, LOCKED); + if (!locked) + return false; + + vma_size = get_value_for_name(addr, SIZE); + vma_rss = get_value_for_name(addr, RSS); + + /* only one page is faulted in */ + return (vma_rss < vma_size); } #define PRESENT_BIT 0x8000000000000000ULL #define PFN_MASK 0x007FFFFFFFFFFFFFULL #define UNEVICTABLE_BIT (1UL << 18) -static int lock_check(char *map) +static int lock_check(unsigned long addr) { - unsigned long page_size = getpagesize(); - uint64_t page1_flags, page2_flags; + bool locked; + unsigned long vma_size, vma_rss; - page1_flags = get_pageflags((unsigned long)map); - page2_flags = get_pageflags((unsigned long)map + page_size); + locked = is_vmflag_set(addr, LOCKED); + if (!locked) + return false; - /* Both pages should be present */ - if (((page1_flags & PRESENT_BIT) == 0) || - ((page2_flags & PRESENT_BIT) == 0)) { - printf("Failed to make both pages present\n"); - return 1; - } + vma_size = get_value_for_name(addr, SIZE); + vma_rss = get_value_for_name(addr, RSS); - page1_flags = get_kpageflags(page1_flags & PFN_MASK); - page2_flags = get_kpageflags(page2_flags & PFN_MASK); - - /* Both pages should be unevictable */ - if (((page1_flags & UNEVICTABLE_BIT) == 0) || - ((page2_flags & UNEVICTABLE_BIT) == 0)) { - printf("Failed to make both pages unevictable\n"); - return 1; - } - - if (!is_vmflag_set((unsigned long)map, LOCKED)) { - printf("VMA flag %s is missing on page 1\n", LOCKED); - return 1; - } - - if (!is_vmflag_set((unsigned long)map + page_size, LOCKED)) { - printf("VMA flag %s is missing on page 2\n", LOCKED); - return 1; - } - - return 0; + return (vma_rss == vma_size); } static int unlock_lock_check(char *map) { - unsigned long page_size = getpagesize(); - uint64_t page1_flags, page2_flags; - - page1_flags = get_pageflags((unsigned long)map); - page2_flags = get_pageflags((unsigned long)map + page_size); - page1_flags = get_kpageflags(page1_flags & PFN_MASK); - page2_flags = get_kpageflags(page2_flags & PFN_MASK); - - if ((page1_flags & UNEVICTABLE_BIT) || (page2_flags & UNEVICTABLE_BIT)) { - printf("A page is still marked unevictable after unlock\n"); - return 1; - } - if (is_vmflag_set((unsigned long)map, LOCKED)) { printf("VMA flag %s is present on page 1 after unlock\n", LOCKED); return 1; } - if (is_vmflag_set((unsigned long)map + page_size, LOCKED)) { - printf("VMA flag %s is present on page 2 after unlock\n", LOCKED); - return 1; - } - return 0; } @@ -311,7 +210,7 @@ static int test_mlock_lock() goto unmap; } - if (lock_check(map)) + if (!lock_check((unsigned long)map)) goto unmap; /* Now unlock and recheck attributes */ @@ -330,64 +229,18 @@ static int test_mlock_lock() static int onfault_check(char *map) { - unsigned long page_size = getpagesize(); - uint64_t page1_flags, page2_flags; - - page1_flags = get_pageflags((unsigned long)map); - page2_flags = get_pageflags((unsigned long)map + page_size); - - /* Neither page should be present */ - if ((page1_flags & PRESENT_BIT) || (page2_flags & PRESENT_BIT)) { - printf("Pages were made present by MLOCK_ONFAULT\n"); - return 1; - } - *map = 'a'; - page1_flags = get_pageflags((unsigned long)map); - page2_flags = get_pageflags((unsigned long)map + page_size); - - /* Only page 1 should be present */ - if ((page1_flags & PRESENT_BIT) == 0) { - printf("Page 1 is not present after fault\n"); - return 1; - } else if (page2_flags & PRESENT_BIT) { - printf("Page 2 was made present\n"); - return 1; - } - - page1_flags = get_kpageflags(page1_flags & PFN_MASK); - - /* Page 1 should be unevictable */ - if ((page1_flags & UNEVICTABLE_BIT) == 0) { - printf("Failed to make faulted page unevictable\n"); - return 1; - } - if (!is_vma_lock_on_fault((unsigned long)map)) { printf("VMA is not marked for lock on fault\n"); return 1; } - if (!is_vma_lock_on_fault((unsigned long)map + page_size)) { - printf("VMA is not marked for lock on fault\n"); - return 1; - } - return 0; } static int unlock_onfault_check(char *map) { unsigned long page_size = getpagesize(); - uint64_t page1_flags; - - page1_flags = get_pageflags((unsigned long)map); - page1_flags = get_kpageflags(page1_flags & PFN_MASK); - - if (page1_flags & UNEVICTABLE_BIT) { - printf("Page 1 is still marked unevictable after unlock\n"); - return 1; - } if (is_vma_lock_on_fault((unsigned long)map) || is_vma_lock_on_fault((unsigned long)map + page_size)) { @@ -445,7 +298,6 @@ static int test_lock_onfault_of_present() char *map; int ret = 1; unsigned long page_size = getpagesize(); - uint64_t page1_flags, page2_flags; map = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); @@ -465,17 +317,6 @@ static int test_lock_onfault_of_present() goto unmap; } - page1_flags = get_pageflags((unsigned long)map); - page2_flags = get_pageflags((unsigned long)map + page_size); - page1_flags = get_kpageflags(page1_flags & PFN_MASK); - page2_flags = get_kpageflags(page2_flags & PFN_MASK); - - /* Page 1 should be unevictable */ - if ((page1_flags & UNEVICTABLE_BIT) == 0) { - printf("Failed to make present page unevictable\n"); - goto unmap; - } - if (!is_vma_lock_on_fault((unsigned long)map) || !is_vma_lock_on_fault((unsigned long)map + page_size)) { printf("VMA with present pages is not marked lock on fault\n"); @@ -507,7 +348,7 @@ static int test_munlockall() goto out; } - if (lock_check(map)) + if (!lock_check((unsigned long)map)) goto unmap; if (munlockall()) { @@ -549,7 +390,7 @@ static int test_munlockall() goto out; } - if (lock_check(map)) + if (!lock_check((unsigned long)map)) goto unmap; if (munlockall()) { diff --git a/tools/testing/selftests/vm/mremap_dontunmap.c b/tools/testing/selftests/vm/mremap_dontunmap.c new file mode 100644 index 000000000000..ee06cb0b9efb --- /dev/null +++ b/tools/testing/selftests/vm/mremap_dontunmap.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Tests for mremap w/ MREMAP_DONTUNMAP. + * + * Copyright 2020, Brian Geffon + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#include "../kselftest.h" + +#ifndef MREMAP_DONTUNMAP +#define MREMAP_DONTUNMAP 4 +#endif + +unsigned long page_size; +char *page_buffer; + +static void dump_maps(void) +{ + char cmd[32]; + + snprintf(cmd, sizeof(cmd), "cat /proc/%d/maps", getpid()); + system(cmd); +} + +#define BUG_ON(condition, description) \ + do { \ + if (condition) { \ + fprintf(stderr, "[FAIL]\t%s():%d\t%s:%s\n", __func__, \ + __LINE__, (description), strerror(errno)); \ + dump_maps(); \ + exit(1); \ + } \ + } while (0) + +// Try a simple operation for to "test" for kernel support this prevents +// reporting tests as failed when it's run on an older kernel. +static int kernel_support_for_mremap_dontunmap() +{ + int ret = 0; + unsigned long num_pages = 1; + void *source_mapping = mmap(NULL, num_pages * page_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + BUG_ON(source_mapping == MAP_FAILED, "mmap"); + + // This simple remap should only fail if MREMAP_DONTUNMAP isn't + // supported. + void *dest_mapping = + mremap(source_mapping, num_pages * page_size, num_pages * page_size, + MREMAP_DONTUNMAP | MREMAP_MAYMOVE, 0); + if (dest_mapping == MAP_FAILED) { + ret = errno; + } else { + BUG_ON(munmap(dest_mapping, num_pages * page_size) == -1, + "unable to unmap destination mapping"); + } + + BUG_ON(munmap(source_mapping, num_pages * page_size) == -1, + "unable to unmap source mapping"); + return ret; +} + +// This helper will just validate that an entire mapping contains the expected +// byte. +static int check_region_contains_byte(void *addr, unsigned long size, char byte) +{ + BUG_ON(size & (page_size - 1), + "check_region_contains_byte expects page multiples"); + BUG_ON((unsigned long)addr & (page_size - 1), + "check_region_contains_byte expects page alignment"); + + memset(page_buffer, byte, page_size); + + unsigned long num_pages = size / page_size; + unsigned long i; + + // Compare each page checking that it contains our expected byte. + for (i = 0; i < num_pages; ++i) { + int ret = + memcmp(addr + (i * page_size), page_buffer, page_size); + if (ret) { + return ret; + } + } + + return 0; +} + +// this test validates that MREMAP_DONTUNMAP moves the pagetables while leaving +// the source mapping mapped. +static void mremap_dontunmap_simple() +{ + unsigned long num_pages = 5; + + void *source_mapping = + mmap(NULL, num_pages * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + BUG_ON(source_mapping == MAP_FAILED, "mmap"); + + memset(source_mapping, 'a', num_pages * page_size); + + // Try to just move the whole mapping anywhere (not fixed). + void *dest_mapping = + mremap(source_mapping, num_pages * page_size, num_pages * page_size, + MREMAP_DONTUNMAP | MREMAP_MAYMOVE, NULL); + BUG_ON(dest_mapping == MAP_FAILED, "mremap"); + + // Validate that the pages have been moved, we know they were moved if + // the dest_mapping contains a's. + BUG_ON(check_region_contains_byte + (dest_mapping, num_pages * page_size, 'a') != 0, + "pages did not migrate"); + BUG_ON(check_region_contains_byte + (source_mapping, num_pages * page_size, 0) != 0, + "source should have no ptes"); + + BUG_ON(munmap(dest_mapping, num_pages * page_size) == -1, + "unable to unmap destination mapping"); + BUG_ON(munmap(source_mapping, num_pages * page_size) == -1, + "unable to unmap source mapping"); +} + +// This test validates MREMAP_DONTUNMAP will move page tables to a specific +// destination using MREMAP_FIXED, also while validating that the source +// remains intact. +static void mremap_dontunmap_simple_fixed() +{ + unsigned long num_pages = 5; + + // Since we want to guarantee that we can remap to a point, we will + // create a mapping up front. + void *dest_mapping = + mmap(NULL, num_pages * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + BUG_ON(dest_mapping == MAP_FAILED, "mmap"); + memset(dest_mapping, 'X', num_pages * page_size); + + void *source_mapping = + mmap(NULL, num_pages * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + BUG_ON(source_mapping == MAP_FAILED, "mmap"); + memset(source_mapping, 'a', num_pages * page_size); + + void *remapped_mapping = + mremap(source_mapping, num_pages * page_size, num_pages * page_size, + MREMAP_FIXED | MREMAP_DONTUNMAP | MREMAP_MAYMOVE, + dest_mapping); + BUG_ON(remapped_mapping == MAP_FAILED, "mremap"); + BUG_ON(remapped_mapping != dest_mapping, + "mremap should have placed the remapped mapping at dest_mapping"); + + // The dest mapping will have been unmap by mremap so we expect the Xs + // to be gone and replaced with a's. + BUG_ON(check_region_contains_byte + (dest_mapping, num_pages * page_size, 'a') != 0, + "pages did not migrate"); + + // And the source mapping will have had its ptes dropped. + BUG_ON(check_region_contains_byte + (source_mapping, num_pages * page_size, 0) != 0, + "source should have no ptes"); + + BUG_ON(munmap(dest_mapping, num_pages * page_size) == -1, + "unable to unmap destination mapping"); + BUG_ON(munmap(source_mapping, num_pages * page_size) == -1, + "unable to unmap source mapping"); +} + +// This test validates that we can MREMAP_DONTUNMAP for a portion of an +// existing mapping. +static void mremap_dontunmap_partial_mapping() +{ + /* + * source mapping: + * -------------- + * | aaaaaaaaaa | + * -------------- + * to become: + * -------------- + * | aaaaa00000 | + * -------------- + * With the destination mapping containing 5 pages of As. + * --------- + * | aaaaa | + * --------- + */ + unsigned long num_pages = 10; + void *source_mapping = + mmap(NULL, num_pages * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + BUG_ON(source_mapping == MAP_FAILED, "mmap"); + memset(source_mapping, 'a', num_pages * page_size); + + // We will grab the last 5 pages of the source and move them. + void *dest_mapping = + mremap(source_mapping + (5 * page_size), 5 * page_size, + 5 * page_size, + MREMAP_DONTUNMAP | MREMAP_MAYMOVE, NULL); + BUG_ON(dest_mapping == MAP_FAILED, "mremap"); + + // We expect the first 5 pages of the source to contain a's and the + // final 5 pages to contain zeros. + BUG_ON(check_region_contains_byte(source_mapping, 5 * page_size, 'a') != + 0, "first 5 pages of source should have original pages"); + BUG_ON(check_region_contains_byte + (source_mapping + (5 * page_size), 5 * page_size, 0) != 0, + "final 5 pages of source should have no ptes"); + + // Finally we expect the destination to have 5 pages worth of a's. + BUG_ON(check_region_contains_byte(dest_mapping, 5 * page_size, 'a') != + 0, "dest mapping should contain ptes from the source"); + + BUG_ON(munmap(dest_mapping, 5 * page_size) == -1, + "unable to unmap destination mapping"); + BUG_ON(munmap(source_mapping, num_pages * page_size) == -1, + "unable to unmap source mapping"); +} + +// This test validates that we can remap over only a portion of a mapping. +static void mremap_dontunmap_partial_mapping_overwrite(void) +{ + /* + * source mapping: + * --------- + * |aaaaa| + * --------- + * dest mapping initially: + * ----------- + * |XXXXXXXXXX| + * ------------ + * Source to become: + * --------- + * |00000| + * --------- + * With the destination mapping containing 5 pages of As. + * ------------ + * |aaaaaXXXXX| + * ------------ + */ + void *source_mapping = + mmap(NULL, 5 * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + BUG_ON(source_mapping == MAP_FAILED, "mmap"); + memset(source_mapping, 'a', 5 * page_size); + + void *dest_mapping = + mmap(NULL, 10 * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + BUG_ON(dest_mapping == MAP_FAILED, "mmap"); + memset(dest_mapping, 'X', 10 * page_size); + + // We will grab the last 5 pages of the source and move them. + void *remapped_mapping = + mremap(source_mapping, 5 * page_size, + 5 * page_size, + MREMAP_DONTUNMAP | MREMAP_MAYMOVE | MREMAP_FIXED, dest_mapping); + BUG_ON(dest_mapping == MAP_FAILED, "mremap"); + BUG_ON(dest_mapping != remapped_mapping, "expected to remap to dest_mapping"); + + BUG_ON(check_region_contains_byte(source_mapping, 5 * page_size, 0) != + 0, "first 5 pages of source should have no ptes"); + + // Finally we expect the destination to have 5 pages worth of a's. + BUG_ON(check_region_contains_byte(dest_mapping, 5 * page_size, 'a') != 0, + "dest mapping should contain ptes from the source"); + + // Finally the last 5 pages shouldn't have been touched. + BUG_ON(check_region_contains_byte(dest_mapping + (5 * page_size), + 5 * page_size, 'X') != 0, + "dest mapping should have retained the last 5 pages"); + + BUG_ON(munmap(dest_mapping, 10 * page_size) == -1, + "unable to unmap destination mapping"); + BUG_ON(munmap(source_mapping, 5 * page_size) == -1, + "unable to unmap source mapping"); +} + +int main(void) +{ + page_size = sysconf(_SC_PAGE_SIZE); + + // test for kernel support for MREMAP_DONTUNMAP skipping the test if + // not. + if (kernel_support_for_mremap_dontunmap() != 0) { + printf("No kernel support for MREMAP_DONTUNMAP\n"); + return KSFT_SKIP; + } + + // Keep a page sized buffer around for when we need it. + page_buffer = + mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + BUG_ON(page_buffer == MAP_FAILED, "unable to mmap a page."); + + mremap_dontunmap_simple(); + mremap_dontunmap_simple_fixed(); + mremap_dontunmap_partial_mapping(); + mremap_dontunmap_partial_mapping_overwrite(); + + BUG_ON(munmap(page_buffer, page_size) == -1, + "unable to unmap page buffer"); + + printf("OK\n"); + return 0; +} diff --git a/tools/testing/selftests/vm/run_vmtests b/tools/testing/selftests/vm/run_vmtests index f33714843198..665009ebfba4 100755 --- a/tools/testing/selftests/vm/run_vmtests +++ b/tools/testing/selftests/vm/run_vmtests @@ -123,6 +123,28 @@ else echo "[PASS]" fi +echo "--------------------------------------------" +echo "running 'gup_benchmark -U' (normal/slow gup)" +echo "--------------------------------------------" +./gup_benchmark -U +if [ $? -ne 0 ]; then + echo "[FAIL]" + exitcode=1 +else + echo "[PASS]" +fi + +echo "------------------------------------------" +echo "running gup_benchmark -b (pin_user_pages)" +echo "------------------------------------------" +./gup_benchmark -b +if [ $? -ne 0 ]; then + echo "[FAIL]" + exitcode=1 +else + echo "[PASS]" +fi + echo "-------------------" echo "running userfaultfd" echo "-------------------" @@ -270,4 +292,19 @@ else exitcode=1 fi +echo "------------------------------------" +echo "running MREMAP_DONTUNMAP smoke test" +echo "------------------------------------" +./mremap_dontunmap +ret_val=$? + +if [ $ret_val -eq 0 ]; then + echo "[PASS]" +elif [ $ret_val -eq $ksft_skip ]; then + echo "[SKIP]" + exitcode=$ksft_skip +else + echo "[FAIL]" + exitcode=1 +fi exit $exitcode diff --git a/tools/testing/selftests/vm/write_hugetlb_memory.sh b/tools/testing/selftests/vm/write_hugetlb_memory.sh new file mode 100644 index 000000000000..d3d0d108924d --- /dev/null +++ b/tools/testing/selftests/vm/write_hugetlb_memory.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +set -e + +size=$1 +populate=$2 +write=$3 +cgroup=$4 +path=$5 +method=$6 +private=$7 +want_sleep=$8 +reserve=$9 + +echo "Putting task in cgroup '$cgroup'" +echo $$ > /dev/cgroup/memory/"$cgroup"/cgroup.procs + +echo "Method is $method" + +set +e +./write_to_hugetlbfs -p "$path" -s "$size" "$write" "$populate" -m "$method" \ + "$private" "$want_sleep" "$reserve" diff --git a/tools/testing/selftests/vm/write_to_hugetlbfs.c b/tools/testing/selftests/vm/write_to_hugetlbfs.c new file mode 100644 index 000000000000..110bc4e4015d --- /dev/null +++ b/tools/testing/selftests/vm/write_to_hugetlbfs.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This program reserves and uses hugetlb memory, supporting a bunch of + * scenarios needed by the charged_reserved_hugetlb.sh test. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Global definitions. */ +enum method { + HUGETLBFS, + MMAP_MAP_HUGETLB, + SHM, + MAX_METHOD +}; + + +/* Global variables. */ +static const char *self; +static char *shmaddr; +static int shmid; + +/* + * Show usage and exit. + */ +static void exit_usage(void) +{ + printf("Usage: %s -p -s " + "[-m <0=hugetlbfs | 1=mmap(MAP_HUGETLB)>] [-l] [-r] " + "[-o] [-w] [-n]\n", + self); + exit(EXIT_FAILURE); +} + +void sig_handler(int signo) +{ + printf("Received %d.\n", signo); + if (signo == SIGINT) { + printf("Deleting the memory\n"); + if (shmdt((const void *)shmaddr) != 0) { + perror("Detach failure"); + shmctl(shmid, IPC_RMID, NULL); + exit(4); + } + + shmctl(shmid, IPC_RMID, NULL); + printf("Done deleting the memory\n"); + } + exit(2); +} + +int main(int argc, char **argv) +{ + int fd = 0; + int key = 0; + int *ptr = NULL; + int c = 0; + int size = 0; + char path[256] = ""; + enum method method = MAX_METHOD; + int want_sleep = 0, private = 0; + int populate = 0; + int write = 0; + int reserve = 1; + + unsigned long i; + + if (signal(SIGINT, sig_handler) == SIG_ERR) + err(1, "\ncan't catch SIGINT\n"); + + /* Parse command-line arguments. */ + setvbuf(stdout, NULL, _IONBF, 0); + self = argv[0]; + + while ((c = getopt(argc, argv, "s:p:m:owlrn")) != -1) { + switch (c) { + case 's': + size = atoi(optarg); + break; + case 'p': + strncpy(path, optarg, sizeof(path)); + break; + case 'm': + if (atoi(optarg) >= MAX_METHOD) { + errno = EINVAL; + perror("Invalid -m."); + exit_usage(); + } + method = atoi(optarg); + break; + case 'o': + populate = 1; + break; + case 'w': + write = 1; + break; + case 'l': + want_sleep = 1; + break; + case 'r': + private + = 1; + break; + case 'n': + reserve = 0; + break; + default: + errno = EINVAL; + perror("Invalid arg"); + exit_usage(); + } + } + + if (strncmp(path, "", sizeof(path)) != 0) { + printf("Writing to this path: %s\n", path); + } else { + errno = EINVAL; + perror("path not found"); + exit_usage(); + } + + if (size != 0) { + printf("Writing this size: %d\n", size); + } else { + errno = EINVAL; + perror("size not found"); + exit_usage(); + } + + if (!populate) + printf("Not populating.\n"); + else + printf("Populating.\n"); + + if (!write) + printf("Not writing to memory.\n"); + + if (method == MAX_METHOD) { + errno = EINVAL; + perror("-m Invalid"); + exit_usage(); + } else + printf("Using method=%d\n", method); + + if (!private) + printf("Shared mapping.\n"); + else + printf("Private mapping.\n"); + + if (!reserve) + printf("NO_RESERVE mapping.\n"); + else + printf("RESERVE mapping.\n"); + + switch (method) { + case HUGETLBFS: + printf("Allocating using HUGETLBFS.\n"); + fd = open(path, O_CREAT | O_RDWR, 0777); + if (fd == -1) + err(1, "Failed to open file."); + + ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + (private ? MAP_PRIVATE : MAP_SHARED) | + (populate ? MAP_POPULATE : 0) | + (reserve ? 0 : MAP_NORESERVE), + fd, 0); + + if (ptr == MAP_FAILED) { + close(fd); + err(1, "Error mapping the file"); + } + break; + case MMAP_MAP_HUGETLB: + printf("Allocating using MAP_HUGETLB.\n"); + ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + (private ? (MAP_PRIVATE | MAP_ANONYMOUS) : + MAP_SHARED) | + MAP_HUGETLB | (populate ? MAP_POPULATE : 0) | + (reserve ? 0 : MAP_NORESERVE), + -1, 0); + + if (ptr == MAP_FAILED) + err(1, "mmap"); + + printf("Returned address is %p\n", ptr); + break; + case SHM: + printf("Allocating using SHM.\n"); + shmid = shmget(key, size, + SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); + if (shmid < 0) { + shmid = shmget(++key, size, + SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); + if (shmid < 0) + err(1, "shmget"); + } + printf("shmid: 0x%x, shmget key:%d\n", shmid, key); + + ptr = shmat(shmid, NULL, 0); + if (ptr == (int *)-1) { + perror("Shared memory attach failure"); + shmctl(shmid, IPC_RMID, NULL); + exit(2); + } + printf("shmaddr: %p\n", ptr); + + break; + default: + errno = EINVAL; + err(1, "Invalid method."); + } + + if (write) { + printf("Writing to memory.\n"); + memset(ptr, 1, size); + } + + if (want_sleep) { + /* Signal to caller that we're done. */ + printf("DONE\n"); + + /* Hold memory until external kill signal is delivered. */ + while (1) + sleep(100); + } + + if (method == HUGETLBFS) + close(fd); + + return 0; +} diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c index 0d9438e9de2a..93bd59b46848 100644 --- a/virt/kvm/arm/arch_timer.c +++ b/virt/kvm/arm/arch_timer.c @@ -788,7 +788,7 @@ u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid) vcpu_ptimer(vcpu), TIMER_REG_CTL); case KVM_REG_ARM_PTIMER_CNT: return kvm_arm_timer_read(vcpu, - vcpu_vtimer(vcpu), TIMER_REG_CNT); + vcpu_ptimer(vcpu), TIMER_REG_CNT); case KVM_REG_ARM_PTIMER_CVAL: return kvm_arm_timer_read(vcpu, vcpu_ptimer(vcpu), TIMER_REG_CVAL); diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c index eda7b624eab8..48d0ec44ad77 100644 --- a/virt/kvm/arm/arm.c +++ b/virt/kvm/arm/arm.c @@ -64,12 +64,12 @@ int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; } -int kvm_arch_hardware_setup(void) +int kvm_arch_hardware_setup(void *opaque) { return 0; } -int kvm_arch_check_processor_compat(void) +int kvm_arch_check_processor_compat(void *opaque) { return 0; } @@ -625,6 +625,14 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) kvm_update_stolen_time(vcpu); + + if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) { + /* The distributor enable bits were changed */ + preempt_disable(); + vgic_v4_put(vcpu, false); + vgic_v4_load(vcpu); + preempt_enable(); + } } } @@ -1181,55 +1189,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp, return r; } -/** - * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot - * @kvm: kvm instance - * @log: slot id and address to which we copy the log - * - * Steps 1-4 below provide general overview of dirty page logging. See - * kvm_get_dirty_log_protect() function description for additional details. - * - * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we - * always flush the TLB (step 4) even if previous step failed and the dirty - * bitmap may be corrupt. Regardless of previous outcome the KVM logging API - * does not preclude user space subsequent dirty log read. Flushing TLB ensures - * writes will be marked dirty for next log read. - * - * 1. Take a snapshot of the bit and clear it if needed. - * 2. Write protect the corresponding page. - * 3. Copy the snapshot to the userspace. - * 4. Flush TLB's if needed. - */ -int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { - bool flush = false; - int r; - mutex_lock(&kvm->slots_lock); - - r = kvm_get_dirty_log_protect(kvm, log, &flush); - - if (flush) - kvm_flush_remote_tlbs(kvm); - - mutex_unlock(&kvm->slots_lock); - return r; } -int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log) +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) { - bool flush = false; - int r; - - mutex_lock(&kvm->slots_lock); - - r = kvm_clear_dirty_log_protect(kvm, log, &flush); - - if (flush) - kvm_flush_remote_tlbs(kvm); - - mutex_unlock(&kvm->slots_lock); - return r; + kvm_flush_remote_tlbs(kvm); } static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm, diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c index 19c961ac4e3c..e3b9ee268823 100644 --- a/virt/kvm/arm/mmu.c +++ b/virt/kvm/arm/mmu.c @@ -1534,8 +1534,13 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot) { struct kvm_memslots *slots = kvm_memslots(kvm); struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); - phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; - phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + phys_addr_t start, end; + + if (WARN_ON_ONCE(!memslot)) + return; + + start = memslot->base_gfn << PAGE_SHIFT; + end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; spin_lock(&kvm->mmu_lock); stage2_wp_range(kvm, start, end); @@ -2251,7 +2256,7 @@ int kvm_mmu_init(void) void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { @@ -2349,17 +2354,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, return ret; } -void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) +void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { } -int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned long npages) -{ - return 0; -} - void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) { } diff --git a/virt/kvm/arm/psci.c b/virt/kvm/arm/psci.c index 17e2bdd4b76f..14a162e295a9 100644 --- a/virt/kvm/arm/psci.c +++ b/virt/kvm/arm/psci.c @@ -12,7 +12,6 @@ #include #include -#include #include #include diff --git a/virt/kvm/arm/vgic/vgic-debug.c b/virt/kvm/arm/vgic/vgic-debug.c index cc12fe9b2df3..b13a9e3f99dd 100644 --- a/virt/kvm/arm/vgic/vgic-debug.c +++ b/virt/kvm/arm/vgic/vgic-debug.c @@ -178,6 +178,8 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq, struct kvm_vcpu *vcpu) { char *type; + bool pending; + if (irq->intid < VGIC_NR_SGIS) type = "SGI"; else if (irq->intid < VGIC_NR_PRIVATE_IRQS) @@ -190,6 +192,16 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq, if (irq->intid ==0 || irq->intid == VGIC_NR_PRIVATE_IRQS) print_header(s, irq, vcpu); + pending = irq->pending_latch; + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + int err; + + err = irq_get_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + &pending); + WARN_ON_ONCE(err); + } + seq_printf(s, " %s %4d " " %2d " "%d%d%d%d%d%d%d " @@ -201,7 +213,7 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq, "\n", type, irq->intid, (irq->target_vcpu) ? irq->target_vcpu->vcpu_id : -1, - irq->pending_latch, + pending, irq->line_level, irq->active, irq->enabled, diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c index ebc218840fc2..e72dcc454247 100644 --- a/virt/kvm/arm/vgic/vgic-mmio-v3.c +++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c @@ -3,9 +3,11 @@ * VGICv3 MMIO handling functions */ +#include #include #include #include +#include #include #include @@ -69,6 +71,8 @@ static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, if (vgic->enabled) value |= GICD_CTLR_ENABLE_SS_G1; value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS; + if (vgic->nassgireq) + value |= GICD_CTLR_nASSGIreq; break; case GICD_TYPER: value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS; @@ -80,6 +84,10 @@ static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19; } break; + case GICD_TYPER2: + if (kvm_vgic_global_state.has_gicv4_1) + value = GICD_TYPER2_nASSGIcap; + break; case GICD_IIDR: value = (PRODUCT_ID_KVM << GICD_IIDR_PRODUCT_ID_SHIFT) | (vgic->implementation_rev << GICD_IIDR_REVISION_SHIFT) | @@ -97,17 +105,46 @@ static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu, unsigned long val) { struct vgic_dist *dist = &vcpu->kvm->arch.vgic; - bool was_enabled = dist->enabled; switch (addr & 0x0c) { - case GICD_CTLR: + case GICD_CTLR: { + bool was_enabled, is_hwsgi; + + mutex_lock(&vcpu->kvm->lock); + + was_enabled = dist->enabled; + is_hwsgi = dist->nassgireq; + dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; - if (!was_enabled && dist->enabled) + /* Not a GICv4.1? No HW SGIs */ + if (!kvm_vgic_global_state.has_gicv4_1) + val &= ~GICD_CTLR_nASSGIreq; + + /* Dist stays enabled? nASSGIreq is RO */ + if (was_enabled && dist->enabled) { + val &= ~GICD_CTLR_nASSGIreq; + val |= FIELD_PREP(GICD_CTLR_nASSGIreq, is_hwsgi); + } + + /* Switching HW SGIs? */ + dist->nassgireq = val & GICD_CTLR_nASSGIreq; + if (is_hwsgi != dist->nassgireq) + vgic_v4_configure_vsgis(vcpu->kvm); + + if (kvm_vgic_global_state.has_gicv4_1 && + was_enabled != dist->enabled) + kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4); + else if (!was_enabled && dist->enabled) vgic_kick_vcpus(vcpu->kvm); + + mutex_unlock(&vcpu->kvm->lock); break; + } case GICD_TYPER: + case GICD_TYPER2: case GICD_IIDR: + /* This is at best for documentation purposes... */ return; } } @@ -116,10 +153,22 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { + struct vgic_dist *dist = &vcpu->kvm->arch.vgic; + switch (addr & 0x0c) { + case GICD_TYPER2: case GICD_IIDR: if (val != vgic_mmio_read_v3_misc(vcpu, addr, len)) return -EINVAL; + return 0; + case GICD_CTLR: + /* Not a GICv4.1? No HW SGIs */ + if (!kvm_vgic_global_state.has_gicv4_1) + val &= ~GICD_CTLR_nASSGIreq; + + dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; + dist->nassgireq = val & GICD_CTLR_nASSGIreq; + return 0; } vgic_mmio_write_v3_misc(vcpu, addr, len, val); @@ -257,8 +306,18 @@ static unsigned long vgic_v3_uaccess_read_pending(struct kvm_vcpu *vcpu, */ for (i = 0; i < len * 8; i++) { struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); + bool state = irq->pending_latch; - if (irq->pending_latch) + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + int err; + + err = irq_get_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + &state); + WARN_ON(err); + } + + if (state) value |= (1U << i); vgic_put_irq(vcpu->kvm, irq); @@ -942,8 +1001,18 @@ void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1) * generate interrupts of either group. */ if (!irq->group || allow_group1) { - irq->pending_latch = true; - vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + if (!irq->hw) { + irq->pending_latch = true; + vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + } else { + /* HW SGI? Ask the GIC to inject it */ + int err; + err = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + true); + WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + } } else { raw_spin_unlock_irqrestore(&irq->irq_lock, flags); } diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c index 97fb2a40e6ba..2199302597fa 100644 --- a/virt/kvm/arm/vgic/vgic-mmio.c +++ b/virt/kvm/arm/vgic/vgic-mmio.c @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include #include @@ -59,6 +61,11 @@ unsigned long vgic_mmio_read_group(struct kvm_vcpu *vcpu, return value; } +static void vgic_update_vsgi(struct vgic_irq *irq) +{ + WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group)); +} + void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { @@ -71,7 +78,12 @@ void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr, raw_spin_lock_irqsave(&irq->irq_lock, flags); irq->group = !!(val & BIT(i)); - vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + vgic_update_vsgi(irq); + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + } else { + vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + } vgic_put_irq(vcpu->kvm, irq); } @@ -113,7 +125,21 @@ void vgic_mmio_write_senable(struct kvm_vcpu *vcpu, struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (vgic_irq_is_mapped_level(irq)) { + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + if (!irq->enabled) { + struct irq_data *data; + + irq->enabled = true; + data = &irq_to_desc(irq->host_irq)->irq_data; + while (irqd_irq_disabled(data)) + enable_irq(irq->host_irq); + } + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + + continue; + } else if (vgic_irq_is_mapped_level(irq)) { bool was_high = irq->line_level; /* @@ -148,6 +174,8 @@ void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu, struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); raw_spin_lock_irqsave(&irq->irq_lock, flags); + if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled) + disable_irq_nosync(irq->host_irq); irq->enabled = false; @@ -167,10 +195,22 @@ unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu, for (i = 0; i < len * 8; i++) { struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); unsigned long flags; + bool val; raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (irq_is_pending(irq)) - value |= (1U << i); + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + int err; + + val = false; + err = irq_get_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + &val); + WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); + } else { + val = irq_is_pending(irq); + } + + value |= ((u32)val << i); raw_spin_unlock_irqrestore(&irq->irq_lock, flags); vgic_put_irq(vcpu->kvm, irq); @@ -215,6 +255,21 @@ void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, } raw_spin_lock_irqsave(&irq->irq_lock, flags); + + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + /* HW SGI? Ask the GIC to inject it */ + int err; + err = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + true); + WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + + continue; + } + if (irq->hw) vgic_hw_irq_spending(vcpu, irq, is_uaccess); else @@ -269,6 +324,20 @@ void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, raw_spin_lock_irqsave(&irq->irq_lock, flags); + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + /* HW SGI? Ask the GIC to clear its pending bit */ + int err; + err = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + false); + WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + + continue; + } + if (irq->hw) vgic_hw_irq_cpending(vcpu, irq, is_uaccess); else @@ -318,8 +387,15 @@ static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq, raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (irq->hw) { + if (irq->hw && !vgic_irq_is_sgi(irq->intid)) { vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu); + } else if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + /* + * GICv4.1 VSGI feature doesn't track an active state, + * so let's not kid ourselves, there is nothing we can + * do here. + */ + irq->active = false; } else { u32 model = vcpu->kvm->arch.vgic.vgic_model; u8 active_source; @@ -493,6 +569,8 @@ void vgic_mmio_write_priority(struct kvm_vcpu *vcpu, raw_spin_lock_irqsave(&irq->irq_lock, flags); /* Narrow the priority range to what we actually support */ irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS); + if (irq->hw && vgic_irq_is_sgi(irq->intid)) + vgic_update_vsgi(irq); raw_spin_unlock_irqrestore(&irq->irq_lock, flags); vgic_put_irq(vcpu->kvm, irq); diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c index 1bc09b523486..2c9fc13e2c59 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/virt/kvm/arm/vgic/vgic-v3.c @@ -540,6 +540,8 @@ int vgic_v3_map_resources(struct kvm *kvm) goto out; } + if (kvm_vgic_global_state.has_gicv4_1) + vgic_v4_configure_vsgis(kvm); dist->ready = true; out: diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 1eb0f8c76219..27ac833e5ec7 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -97,6 +97,104 @@ static irqreturn_t vgic_v4_doorbell_handler(int irq, void *info) return IRQ_HANDLED; } +static void vgic_v4_sync_sgi_config(struct its_vpe *vpe, struct vgic_irq *irq) +{ + vpe->sgi_config[irq->intid].enabled = irq->enabled; + vpe->sgi_config[irq->intid].group = irq->group; + vpe->sgi_config[irq->intid].priority = irq->priority; +} + +static void vgic_v4_enable_vsgis(struct kvm_vcpu *vcpu) +{ + struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; + int i; + + /* + * With GICv4.1, every virtual SGI can be directly injected. So + * let's pretend that they are HW interrupts, tied to a host + * IRQ. The SGI code will do its magic. + */ + for (i = 0; i < VGIC_NR_SGIS; i++) { + struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, i); + struct irq_desc *desc; + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&irq->irq_lock, flags); + + if (irq->hw) + goto unlock; + + irq->hw = true; + irq->host_irq = irq_find_mapping(vpe->sgi_domain, i); + + /* Transfer the full irq state to the vPE */ + vgic_v4_sync_sgi_config(vpe, irq); + desc = irq_to_desc(irq->host_irq); + ret = irq_domain_activate_irq(irq_desc_get_irq_data(desc), + false); + if (!WARN_ON(ret)) { + /* Transfer pending state */ + ret = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + irq->pending_latch); + WARN_ON(ret); + irq->pending_latch = false; + } + unlock: + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + } +} + +static void vgic_v4_disable_vsgis(struct kvm_vcpu *vcpu) +{ + int i; + + for (i = 0; i < VGIC_NR_SGIS; i++) { + struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, i); + struct irq_desc *desc; + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&irq->irq_lock, flags); + + if (!irq->hw) + goto unlock; + + irq->hw = false; + ret = irq_get_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + &irq->pending_latch); + WARN_ON(ret); + + desc = irq_to_desc(irq->host_irq); + irq_domain_deactivate_irq(irq_desc_get_irq_data(desc)); + unlock: + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + } +} + +/* Must be called with the kvm lock held */ +void vgic_v4_configure_vsgis(struct kvm *kvm) +{ + struct vgic_dist *dist = &kvm->arch.vgic; + struct kvm_vcpu *vcpu; + int i; + + kvm_arm_halt_guest(kvm); + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (dist->nassgireq) + vgic_v4_enable_vsgis(vcpu); + else + vgic_v4_disable_vsgis(vcpu); + } + + kvm_arm_resume_guest(kvm); +} + /** * vgic_v4_init - Initialize the GICv4 data structures * @kvm: Pointer to the VM being initialized @@ -141,6 +239,7 @@ int vgic_v4_init(struct kvm *kvm) kvm_for_each_vcpu(i, vcpu, kvm) { int irq = dist->its_vm.vpes[i]->irq; + unsigned long irq_flags = DB_IRQ_FLAGS; /* * Don't automatically enable the doorbell, as we're @@ -148,8 +247,14 @@ int vgic_v4_init(struct kvm *kvm) * blocked. Also disable the lazy disabling, as the * doorbell could kick us out of the guest too * early... + * + * On GICv4.1, the doorbell is managed in HW and must + * be left enabled. */ - irq_set_status_flags(irq, DB_IRQ_FLAGS); + if (kvm_vgic_global_state.has_gicv4_1) + irq_flags &= ~IRQ_NOAUTOEN; + irq_set_status_flags(irq, irq_flags); + ret = request_irq(irq, vgic_v4_doorbell_handler, 0, "vcpu", vcpu); if (ret) { diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h index c7fefd6b1c80..769e4802645e 100644 --- a/virt/kvm/arm/vgic/vgic.h +++ b/virt/kvm/arm/vgic/vgic.h @@ -316,5 +316,6 @@ void vgic_its_invalidate_cache(struct kvm *kvm); bool vgic_supports_direct_msis(struct kvm *kvm); int vgic_v4_init(struct kvm *kvm); void vgic_v4_teardown(struct kvm *kvm); +void vgic_v4_configure_vsgis(struct kvm *kvm); #endif diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 70f03ce0e5c1..74bdb7bf3295 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -149,8 +149,6 @@ static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn); __visible bool kvm_rebooting; EXPORT_SYMBOL_GPL(kvm_rebooting); -static bool largepages_enabled = true; - #define KVM_EVENT_CREATE_VM 0 #define KVM_EVENT_DESTROY_VM 1 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm); @@ -566,7 +564,7 @@ static struct kvm_memslots *kvm_alloc_memslots(void) return NULL; for (i = 0; i < KVM_MEM_SLOTS_NUM; i++) - slots->id_to_index[i] = slots->memslots[i].id = i; + slots->id_to_index[i] = -1; return slots; } @@ -580,18 +578,14 @@ static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot) memslot->dirty_bitmap = NULL; } -/* - * Free any memory in @free but not in @dont. - */ -static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) +static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { - if (!dont || free->dirty_bitmap != dont->dirty_bitmap) - kvm_destroy_dirty_bitmap(free); + kvm_destroy_dirty_bitmap(slot); - kvm_arch_free_memslot(kvm, free, dont); + kvm_arch_free_memslot(kvm, slot); - free->npages = 0; + slot->flags = 0; + slot->npages = 0; } static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) @@ -602,7 +596,7 @@ static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) return; kvm_for_each_memslot(memslot, slots) - kvm_free_memslot(kvm, memslot, NULL); + kvm_free_memslot(kvm, memslot); kvfree(slots); } @@ -860,9 +854,9 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) /* * Allocation size is twice as large as the actual dirty bitmap size. - * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed. + * See kvm_vm_ioctl_get_dirty_log() why this is needed. */ -static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) +static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot) { unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); @@ -874,63 +868,165 @@ static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot) } /* - * Insert memslot and re-sort memslots based on their GFN, - * so binary search could be used to lookup GFN. - * Sorting algorithm takes advantage of having initially - * sorted array and known changed memslot position. + * Delete a memslot by decrementing the number of used slots and shifting all + * other entries in the array forward one spot. */ -static void update_memslots(struct kvm_memslots *slots, - struct kvm_memory_slot *new, - enum kvm_mr_change change) +static inline void kvm_memslot_delete(struct kvm_memslots *slots, + struct kvm_memory_slot *memslot) { - int id = new->id; - int i = slots->id_to_index[id]; struct kvm_memory_slot *mslots = slots->memslots; + int i; - WARN_ON(mslots[i].id != id); - switch (change) { - case KVM_MR_CREATE: - slots->used_slots++; - WARN_ON(mslots[i].npages || !new->npages); - break; - case KVM_MR_DELETE: - slots->used_slots--; - WARN_ON(new->npages || !mslots[i].npages); - break; - default: - break; - } + if (WARN_ON(slots->id_to_index[memslot->id] == -1)) + return; - while (i < KVM_MEM_SLOTS_NUM - 1 && - new->base_gfn <= mslots[i + 1].base_gfn) { - if (!mslots[i + 1].npages) - break; + slots->used_slots--; + + if (atomic_read(&slots->lru_slot) >= slots->used_slots) + atomic_set(&slots->lru_slot, 0); + + for (i = slots->id_to_index[memslot->id]; i < slots->used_slots; i++) { mslots[i] = mslots[i + 1]; slots->id_to_index[mslots[i].id] = i; - i++; } + mslots[i] = *memslot; + slots->id_to_index[memslot->id] = -1; +} + +/* + * "Insert" a new memslot by incrementing the number of used slots. Returns + * the new slot's initial index into the memslots array. + */ +static inline int kvm_memslot_insert_back(struct kvm_memslots *slots) +{ + return slots->used_slots++; +} + +/* + * Move a changed memslot backwards in the array by shifting existing slots + * with a higher GFN toward the front of the array. Note, the changed memslot + * itself is not preserved in the array, i.e. not swapped at this time, only + * its new index into the array is tracked. Returns the changed memslot's + * current index into the memslots array. + */ +static inline int kvm_memslot_move_backward(struct kvm_memslots *slots, + struct kvm_memory_slot *memslot) +{ + struct kvm_memory_slot *mslots = slots->memslots; + int i; + + if (WARN_ON_ONCE(slots->id_to_index[memslot->id] == -1) || + WARN_ON_ONCE(!slots->used_slots)) + return -1; /* - * The ">=" is needed when creating a slot with base_gfn == 0, - * so that it moves before all those with base_gfn == npages == 0. - * - * On the other hand, if new->npages is zero, the above loop has - * already left i pointing to the beginning of the empty part of - * mslots, and the ">=" would move the hole backwards in this - * case---which is wrong. So skip the loop when deleting a slot. + * Move the target memslot backward in the array by shifting existing + * memslots with a higher GFN (than the target memslot) towards the + * front of the array. */ - if (new->npages) { - while (i > 0 && - new->base_gfn >= mslots[i - 1].base_gfn) { - mslots[i] = mslots[i - 1]; - slots->id_to_index[mslots[i].id] = i; - i--; - } - } else - WARN_ON_ONCE(i != slots->used_slots); + for (i = slots->id_to_index[memslot->id]; i < slots->used_slots - 1; i++) { + if (memslot->base_gfn > mslots[i + 1].base_gfn) + break; - mslots[i] = *new; - slots->id_to_index[mslots[i].id] = i; + WARN_ON_ONCE(memslot->base_gfn == mslots[i + 1].base_gfn); + + /* Shift the next memslot forward one and update its index. */ + mslots[i] = mslots[i + 1]; + slots->id_to_index[mslots[i].id] = i; + } + return i; +} + +/* + * Move a changed memslot forwards in the array by shifting existing slots with + * a lower GFN toward the back of the array. Note, the changed memslot itself + * is not preserved in the array, i.e. not swapped at this time, only its new + * index into the array is tracked. Returns the changed memslot's final index + * into the memslots array. + */ +static inline int kvm_memslot_move_forward(struct kvm_memslots *slots, + struct kvm_memory_slot *memslot, + int start) +{ + struct kvm_memory_slot *mslots = slots->memslots; + int i; + + for (i = start; i > 0; i--) { + if (memslot->base_gfn < mslots[i - 1].base_gfn) + break; + + WARN_ON_ONCE(memslot->base_gfn == mslots[i - 1].base_gfn); + + /* Shift the next memslot back one and update its index. */ + mslots[i] = mslots[i - 1]; + slots->id_to_index[mslots[i].id] = i; + } + return i; +} + +/* + * Re-sort memslots based on their GFN to account for an added, deleted, or + * moved memslot. Sorting memslots by GFN allows using a binary search during + * memslot lookup. + * + * IMPORTANT: Slots are sorted from highest GFN to lowest GFN! I.e. the entry + * at memslots[0] has the highest GFN. + * + * The sorting algorithm takes advantage of having initially sorted memslots + * and knowing the position of the changed memslot. Sorting is also optimized + * by not swapping the updated memslot and instead only shifting other memslots + * and tracking the new index for the update memslot. Only once its final + * index is known is the updated memslot copied into its position in the array. + * + * - When deleting a memslot, the deleted memslot simply needs to be moved to + * the end of the array. + * + * - When creating a memslot, the algorithm "inserts" the new memslot at the + * end of the array and then it forward to its correct location. + * + * - When moving a memslot, the algorithm first moves the updated memslot + * backward to handle the scenario where the memslot's GFN was changed to a + * lower value. update_memslots() then falls through and runs the same flow + * as creating a memslot to move the memslot forward to handle the scenario + * where its GFN was changed to a higher value. + * + * Note, slots are sorted from highest->lowest instead of lowest->highest for + * historical reasons. Originally, invalid memslots where denoted by having + * GFN=0, thus sorting from highest->lowest naturally sorted invalid memslots + * to the end of the array. The current algorithm uses dedicated logic to + * delete a memslot and thus does not rely on invalid memslots having GFN=0. + * + * The other historical motiviation for highest->lowest was to improve the + * performance of memslot lookup. KVM originally used a linear search starting + * at memslots[0]. On x86, the largest memslot usually has one of the highest, + * if not *the* highest, GFN, as the bulk of the guest's RAM is located in a + * single memslot above the 4gb boundary. As the largest memslot is also the + * most likely to be referenced, sorting it to the front of the array was + * advantageous. The current binary search starts from the middle of the array + * and uses an LRU pointer to improve performance for all memslots and GFNs. + */ +static void update_memslots(struct kvm_memslots *slots, + struct kvm_memory_slot *memslot, + enum kvm_mr_change change) +{ + int i; + + if (change == KVM_MR_DELETE) { + kvm_memslot_delete(slots, memslot); + } else { + if (change == KVM_MR_CREATE) + i = kvm_memslot_insert_back(slots); + else + i = kvm_memslot_move_backward(slots, memslot); + i = kvm_memslot_move_forward(slots, memslot, i); + + /* + * Copy the memslot to its new position in memslots and update + * its index accordingly. + */ + slots->memslots[i] = *memslot; + slots->id_to_index[memslot->id] = i; + } } static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem) @@ -983,6 +1079,112 @@ static struct kvm_memslots *install_new_memslots(struct kvm *kvm, return old_memslots; } +/* + * Note, at a minimum, the current number of used slots must be allocated, even + * when deleting a memslot, as we need a complete duplicate of the memslots for + * use when invalidating a memslot prior to deleting/moving the memslot. + */ +static struct kvm_memslots *kvm_dup_memslots(struct kvm_memslots *old, + enum kvm_mr_change change) +{ + struct kvm_memslots *slots; + size_t old_size, new_size; + + old_size = sizeof(struct kvm_memslots) + + (sizeof(struct kvm_memory_slot) * old->used_slots); + + if (change == KVM_MR_CREATE) + new_size = old_size + sizeof(struct kvm_memory_slot); + else + new_size = old_size; + + slots = kvzalloc(new_size, GFP_KERNEL_ACCOUNT); + if (likely(slots)) + memcpy(slots, old, old_size); + + return slots; +} + +static int kvm_set_memslot(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot *old, + struct kvm_memory_slot *new, int as_id, + enum kvm_mr_change change) +{ + struct kvm_memory_slot *slot; + struct kvm_memslots *slots; + int r; + + slots = kvm_dup_memslots(__kvm_memslots(kvm, as_id), change); + if (!slots) + return -ENOMEM; + + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { + /* + * Note, the INVALID flag needs to be in the appropriate entry + * in the freshly allocated memslots, not in @old or @new. + */ + slot = id_to_memslot(slots, old->id); + slot->flags |= KVM_MEMSLOT_INVALID; + + /* + * We can re-use the old memslots, the only difference from the + * newly installed memslots is the invalid flag, which will get + * dropped by update_memslots anyway. We'll also revert to the + * old memslots if preparing the new memory region fails. + */ + slots = install_new_memslots(kvm, as_id, slots); + + /* From this point no new shadow pages pointing to a deleted, + * or moved, memslot will be created. + * + * validation of sp->gfn happens in: + * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) + * - kvm_is_visible_gfn (mmu_check_root) + */ + kvm_arch_flush_shadow_memslot(kvm, slot); + } + + r = kvm_arch_prepare_memory_region(kvm, new, mem, change); + if (r) + goto out_slots; + + update_memslots(slots, new, change); + slots = install_new_memslots(kvm, as_id, slots); + + kvm_arch_commit_memory_region(kvm, mem, old, new, change); + + kvfree(slots); + return 0; + +out_slots: + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) + slots = install_new_memslots(kvm, as_id, slots); + kvfree(slots); + return r; +} + +static int kvm_delete_memslot(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot *old, int as_id) +{ + struct kvm_memory_slot new; + int r; + + if (!old->npages) + return -EINVAL; + + memset(&new, 0, sizeof(new)); + new.id = old->id; + + r = kvm_set_memslot(kvm, mem, old, &new, as_id, KVM_MR_DELETE); + if (r) + return r; + + kvm_free_memslot(kvm, old); + return 0; +} + /* * Allocate some memory and give it an address in the guest physical address * space. @@ -994,162 +1196,118 @@ static struct kvm_memslots *install_new_memslots(struct kvm *kvm, int __kvm_set_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem) { - int r; - gfn_t base_gfn; - unsigned long npages; - struct kvm_memory_slot *slot; struct kvm_memory_slot old, new; - struct kvm_memslots *slots = NULL, *old_memslots; - int as_id, id; + struct kvm_memory_slot *tmp; enum kvm_mr_change change; + int as_id, id; + int r; r = check_memory_region_flags(mem); if (r) - goto out; + return r; - r = -EINVAL; as_id = mem->slot >> 16; id = (u16)mem->slot; /* General sanity checks */ if (mem->memory_size & (PAGE_SIZE - 1)) - goto out; + return -EINVAL; if (mem->guest_phys_addr & (PAGE_SIZE - 1)) - goto out; + return -EINVAL; /* We can read the guest memory with __xxx_user() later on. */ if ((id < KVM_USER_MEM_SLOTS) && ((mem->userspace_addr & (PAGE_SIZE - 1)) || !access_ok((void __user *)(unsigned long)mem->userspace_addr, mem->memory_size))) - goto out; + return -EINVAL; if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) - goto out; + return -EINVAL; if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) - goto out; + return -EINVAL; - slot = id_to_memslot(__kvm_memslots(kvm, as_id), id); - base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; - npages = mem->memory_size >> PAGE_SHIFT; + /* + * Make a full copy of the old memslot, the pointer will become stale + * when the memslots are re-sorted by update_memslots(), and the old + * memslot needs to be referenced after calling update_memslots(), e.g. + * to free its resources and for arch specific behavior. + */ + tmp = id_to_memslot(__kvm_memslots(kvm, as_id), id); + if (tmp) { + old = *tmp; + tmp = NULL; + } else { + memset(&old, 0, sizeof(old)); + old.id = id; + } - if (npages > KVM_MEM_MAX_NR_PAGES) - goto out; - - new = old = *slot; + if (!mem->memory_size) + return kvm_delete_memslot(kvm, mem, &old, as_id); new.id = id; - new.base_gfn = base_gfn; - new.npages = npages; + new.base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; + new.npages = mem->memory_size >> PAGE_SHIFT; new.flags = mem->flags; + new.userspace_addr = mem->userspace_addr; - if (npages) { - if (!old.npages) - change = KVM_MR_CREATE; - else { /* Modify an existing slot. */ - if ((mem->userspace_addr != old.userspace_addr) || - (npages != old.npages) || - ((new.flags ^ old.flags) & KVM_MEM_READONLY)) - goto out; + if (new.npages > KVM_MEM_MAX_NR_PAGES) + return -EINVAL; - if (base_gfn != old.base_gfn) - change = KVM_MR_MOVE; - else if (new.flags != old.flags) - change = KVM_MR_FLAGS_ONLY; - else { /* Nothing to change. */ - r = 0; - goto out; - } - } - } else { - if (!old.npages) - goto out; + if (!old.npages) { + change = KVM_MR_CREATE; + new.dirty_bitmap = NULL; + memset(&new.arch, 0, sizeof(new.arch)); + } else { /* Modify an existing slot. */ + if ((new.userspace_addr != old.userspace_addr) || + (new.npages != old.npages) || + ((new.flags ^ old.flags) & KVM_MEM_READONLY)) + return -EINVAL; - change = KVM_MR_DELETE; - new.base_gfn = 0; - new.flags = 0; + if (new.base_gfn != old.base_gfn) + change = KVM_MR_MOVE; + else if (new.flags != old.flags) + change = KVM_MR_FLAGS_ONLY; + else /* Nothing to change. */ + return 0; + + /* Copy dirty_bitmap and arch from the current memslot. */ + new.dirty_bitmap = old.dirty_bitmap; + memcpy(&new.arch, &old.arch, sizeof(new.arch)); } if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) { /* Check for overlaps */ - r = -EEXIST; - kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) { - if (slot->id == id) + kvm_for_each_memslot(tmp, __kvm_memslots(kvm, as_id)) { + if (tmp->id == id) continue; - if (!((base_gfn + npages <= slot->base_gfn) || - (base_gfn >= slot->base_gfn + slot->npages))) - goto out; + if (!((new.base_gfn + new.npages <= tmp->base_gfn) || + (new.base_gfn >= tmp->base_gfn + tmp->npages))) + return -EEXIST; } } - /* Free page dirty bitmap if unneeded */ + /* Allocate/free page dirty bitmap as needed */ if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) new.dirty_bitmap = NULL; + else if (!new.dirty_bitmap) { + r = kvm_alloc_dirty_bitmap(&new); + if (r) + return r; - r = -ENOMEM; - if (change == KVM_MR_CREATE) { - new.userspace_addr = mem->userspace_addr; - - if (kvm_arch_create_memslot(kvm, &new, npages)) - goto out_free; + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) + bitmap_set(new.dirty_bitmap, 0, new.npages); } - /* Allocate page dirty bitmap if needed */ - if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { - if (kvm_create_dirty_bitmap(&new) < 0) - goto out_free; - } - - slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT); - if (!slots) - goto out_free; - memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots)); - - if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) { - slot = id_to_memslot(slots, id); - slot->flags |= KVM_MEMSLOT_INVALID; - - old_memslots = install_new_memslots(kvm, as_id, slots); - - /* From this point no new shadow pages pointing to a deleted, - * or moved, memslot will be created. - * - * validation of sp->gfn happens in: - * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) - * - kvm_is_visible_gfn (mmu_check_root) - */ - kvm_arch_flush_shadow_memslot(kvm, slot); - - /* - * We can re-use the old_memslots from above, the only difference - * from the currently installed memslots is the invalid flag. This - * will get overwritten by update_memslots anyway. - */ - slots = old_memslots; - } - - r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); + r = kvm_set_memslot(kvm, mem, &old, &new, as_id, change); if (r) - goto out_slots; + goto out_bitmap; - /* actual memory is freed via old in kvm_free_memslot below */ - if (change == KVM_MR_DELETE) { - new.dirty_bitmap = NULL; - memset(&new.arch, 0, sizeof(new.arch)); - } - - update_memslots(slots, &new, change); - old_memslots = install_new_memslots(kvm, as_id, slots); - - kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); - - kvm_free_memslot(kvm, &old, &new); - kvfree(old_memslots); + if (old.dirty_bitmap && !new.dirty_bitmap) + kvm_destroy_dirty_bitmap(&old); return 0; -out_slots: - kvfree(slots); -out_free: - kvm_free_memslot(kvm, &new, &old); -out: +out_bitmap: + if (new.dirty_bitmap && !old.dirty_bitmap) + kvm_destroy_dirty_bitmap(&new); return r; } EXPORT_SYMBOL_GPL(__kvm_set_memory_region); @@ -1175,31 +1333,43 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, return kvm_set_memory_region(kvm, mem); } -int kvm_get_dirty_log(struct kvm *kvm, - struct kvm_dirty_log *log, int *is_dirty) +#ifndef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT +/** + * kvm_get_dirty_log - get a snapshot of dirty pages + * @kvm: pointer to kvm instance + * @log: slot id and address to which we copy the log + * @is_dirty: set to '1' if any dirty pages were found + * @memslot: set to the associated memslot, always valid on success + */ +int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, + int *is_dirty, struct kvm_memory_slot **memslot) { struct kvm_memslots *slots; - struct kvm_memory_slot *memslot; int i, as_id, id; unsigned long n; unsigned long any = 0; + *memslot = NULL; + *is_dirty = 0; + as_id = log->slot >> 16; id = (u16)log->slot; if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS) return -EINVAL; slots = __kvm_memslots(kvm, as_id); - memslot = id_to_memslot(slots, id); - if (!memslot->dirty_bitmap) + *memslot = id_to_memslot(slots, id); + if (!(*memslot) || !(*memslot)->dirty_bitmap) return -ENOENT; - n = kvm_dirty_bitmap_bytes(memslot); + kvm_arch_sync_dirty_log(kvm, *memslot); + + n = kvm_dirty_bitmap_bytes(*memslot); for (i = 0; !any && i < n/sizeof(long); ++i) - any = memslot->dirty_bitmap[i]; + any = (*memslot)->dirty_bitmap[i]; - if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) + if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n)) return -EFAULT; if (any) @@ -1208,13 +1378,12 @@ int kvm_get_dirty_log(struct kvm *kvm, } EXPORT_SYMBOL_GPL(kvm_get_dirty_log); -#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT +#else /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ /** * kvm_get_dirty_log_protect - get a snapshot of dirty pages * and reenable dirty page tracking for the corresponding pages. * @kvm: pointer to kvm instance * @log: slot id and address to which we copy the log - * @flush: true if TLB flush is needed by caller * * We need to keep it in mind that VCPU threads can write to the bitmap * concurrently. So, to avoid losing track of dirty pages we keep the @@ -1231,8 +1400,7 @@ EXPORT_SYMBOL_GPL(kvm_get_dirty_log); * exiting to userspace will be logged for the next call. * */ -int kvm_get_dirty_log_protect(struct kvm *kvm, - struct kvm_dirty_log *log, bool *flush) +static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) { struct kvm_memslots *slots; struct kvm_memory_slot *memslot; @@ -1240,6 +1408,7 @@ int kvm_get_dirty_log_protect(struct kvm *kvm, unsigned long n; unsigned long *dirty_bitmap; unsigned long *dirty_bitmap_buffer; + bool flush; as_id = log->slot >> 16; id = (u16)log->slot; @@ -1248,13 +1417,15 @@ int kvm_get_dirty_log_protect(struct kvm *kvm, slots = __kvm_memslots(kvm, as_id); memslot = id_to_memslot(slots, id); - - dirty_bitmap = memslot->dirty_bitmap; - if (!dirty_bitmap) + if (!memslot || !memslot->dirty_bitmap) return -ENOENT; + dirty_bitmap = memslot->dirty_bitmap; + + kvm_arch_sync_dirty_log(kvm, memslot); + n = kvm_dirty_bitmap_bytes(memslot); - *flush = false; + flush = false; if (kvm->manual_dirty_log_protect) { /* * Unlike kvm_get_dirty_log, we always return false in *flush, @@ -1277,7 +1448,7 @@ int kvm_get_dirty_log_protect(struct kvm *kvm, if (!dirty_bitmap[i]) continue; - *flush = true; + flush = true; mask = xchg(&dirty_bitmap[i], 0); dirty_bitmap_buffer[i] = mask; @@ -1288,21 +1459,55 @@ int kvm_get_dirty_log_protect(struct kvm *kvm, spin_unlock(&kvm->mmu_lock); } + if (flush) + kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); + if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n)) return -EFAULT; return 0; } -EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect); + + +/** + * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot + * @kvm: kvm instance + * @log: slot id and address to which we copy the log + * + * Steps 1-4 below provide general overview of dirty page logging. See + * kvm_get_dirty_log_protect() function description for additional details. + * + * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we + * always flush the TLB (step 4) even if previous step failed and the dirty + * bitmap may be corrupt. Regardless of previous outcome the KVM logging API + * does not preclude user space subsequent dirty log read. Flushing TLB ensures + * writes will be marked dirty for next log read. + * + * 1. Take a snapshot of the bit and clear it if needed. + * 2. Write protect the corresponding page. + * 3. Copy the snapshot to the userspace. + * 4. Flush TLB's if needed. + */ +static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log) +{ + int r; + + mutex_lock(&kvm->slots_lock); + + r = kvm_get_dirty_log_protect(kvm, log); + + mutex_unlock(&kvm->slots_lock); + return r; +} /** * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap * and reenable dirty page tracking for the corresponding pages. * @kvm: pointer to kvm instance * @log: slot id and address from which to fetch the bitmap of dirty pages - * @flush: true if TLB flush is needed by caller */ -int kvm_clear_dirty_log_protect(struct kvm *kvm, - struct kvm_clear_dirty_log *log, bool *flush) +static int kvm_clear_dirty_log_protect(struct kvm *kvm, + struct kvm_clear_dirty_log *log) { struct kvm_memslots *slots; struct kvm_memory_slot *memslot; @@ -1311,6 +1516,7 @@ int kvm_clear_dirty_log_protect(struct kvm *kvm, unsigned long i, n; unsigned long *dirty_bitmap; unsigned long *dirty_bitmap_buffer; + bool flush; as_id = log->slot >> 16; id = (u16)log->slot; @@ -1322,10 +1528,10 @@ int kvm_clear_dirty_log_protect(struct kvm *kvm, slots = __kvm_memslots(kvm, as_id); memslot = id_to_memslot(slots, id); + if (!memslot || !memslot->dirty_bitmap) + return -ENOENT; dirty_bitmap = memslot->dirty_bitmap; - if (!dirty_bitmap) - return -ENOENT; n = ALIGN(log->num_pages, BITS_PER_LONG) / 8; @@ -1334,7 +1540,9 @@ int kvm_clear_dirty_log_protect(struct kvm *kvm, (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63))) return -EINVAL; - *flush = false; + kvm_arch_sync_dirty_log(kvm, memslot); + + flush = false; dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot); if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n)) return -EFAULT; @@ -1357,28 +1565,32 @@ int kvm_clear_dirty_log_protect(struct kvm *kvm, * a problem if userspace sets them in log->dirty_bitmap. */ if (mask) { - *flush = true; + flush = true; kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask); } } spin_unlock(&kvm->mmu_lock); + if (flush) + kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); + return 0; } -EXPORT_SYMBOL_GPL(kvm_clear_dirty_log_protect); -#endif -bool kvm_largepages_enabled(void) +static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, + struct kvm_clear_dirty_log *log) { - return largepages_enabled; -} + int r; -void kvm_disable_largepages(void) -{ - largepages_enabled = false; + mutex_lock(&kvm->slots_lock); + + r = kvm_clear_dirty_log_protect(kvm, log); + + mutex_unlock(&kvm->slots_lock); + return r; } -EXPORT_SYMBOL_GPL(kvm_disable_largepages); +#endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { @@ -1754,12 +1966,6 @@ kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn) } EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic); -kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn) -{ - return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn); -} -EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic); - kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn) { return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn); @@ -3310,9 +3516,6 @@ static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_IOEVENTFD_ANY_LENGTH: case KVM_CAP_CHECK_EXTENSION_VM: case KVM_CAP_ENABLE_CAP_VM: -#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT - case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: -#endif return 1; #ifdef CONFIG_KVM_MMIO case KVM_CAP_COALESCED_MMIO: @@ -3320,6 +3523,10 @@ static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_COALESCED_PIO: return 1; #endif +#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT + case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: + return KVM_DIRTY_LOG_MANUAL_CAPS; +#endif #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING case KVM_CAP_IRQ_ROUTING: return KVM_MAX_IRQ_ROUTES; @@ -3347,11 +3554,17 @@ static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, { switch (cap->cap) { #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT - case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: - if (cap->flags || (cap->args[0] & ~1)) + case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: { + u64 allowed_options = KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE; + + if (cap->args[0] & KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE) + allowed_options = KVM_DIRTY_LOG_MANUAL_CAPS; + + if (cap->flags || (cap->args[0] & ~allowed_options)) return -EINVAL; kvm->manual_dirty_log_protect = cap->args[0]; return 0; + } #endif default: return kvm_vm_ioctl_enable_cap(kvm, cap); @@ -4435,14 +4648,22 @@ struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void) return &kvm_running_vcpu; } -static void check_processor_compat(void *rtn) +struct kvm_cpu_compat_check { + void *opaque; + int *ret; +}; + +static void check_processor_compat(void *data) { - *(int *)rtn = kvm_arch_check_processor_compat(); + struct kvm_cpu_compat_check *c = data; + + *c->ret = kvm_arch_check_processor_compat(c->opaque); } int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, struct module *module) { + struct kvm_cpu_compat_check c; int r; int cpu; @@ -4466,12 +4687,14 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, goto out_free_0; } - r = kvm_arch_hardware_setup(); + r = kvm_arch_hardware_setup(opaque); if (r < 0) goto out_free_1; + c.ret = &r; + c.opaque = opaque; for_each_online_cpu(cpu) { - smp_call_function_single(cpu, check_processor_compat, &r, 1); + smp_call_function_single(cpu, check_processor_compat, &c, 1); if (r < 0) goto out_free_2; }