diff --git a/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml b/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml index c3e14eb6cfff..1cc3304b7414 100644 --- a/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml +++ b/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml @@ -25,6 +25,7 @@ properties: - allwinner,sun50i-a64-dma - allwinner,sun50i-a100-dma - allwinner,sun50i-h6-dma + - allwinner,sun60i-a733-dma - items: - const: allwinner,sun8i-r40-dma - const: allwinner,sun50i-a64-dma @@ -70,6 +71,7 @@ if: - allwinner,sun20i-d1-dma - allwinner,sun50i-a100-dma - allwinner,sun50i-h6-dma + - allwinner,sun60i-a733-dma then: properties: diff --git a/Documentation/devicetree/bindings/dma/amlogic,a9-dma.yaml b/Documentation/devicetree/bindings/dma/amlogic,a9-dma.yaml new file mode 100644 index 000000000000..3e1a4fba3047 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/amlogic,a9-dma.yaml @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/amlogic,a9-dma.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Amlogic general DMA controller + +description: + This is a general-purpose peripheral DMA controller. It currently supports + major peripherals including I2C, I3C, PIO, and CAN-BUS. Transmit and receive + for the same peripheral use two separate channels, controlled by different + register sets. I2C and I3C transfer data in 1-byte units, while PIO and + CAN-BUS transfer data in 4-byte units. From the controller’s perspective, + there is no significant difference. + +maintainers: + - Xianwei Zhao + +properties: + compatible: + const: amlogic,a9-dma + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + const: sys + + '#dma-cells': + const: 2 + description: + The first cell is the DMA channel type(AMLOGIC_A9_DMA_TX or AMLOGIC_A9_DMA_RX). + The second cell is the DMA channel index. + + dma-channels: + maximum: 64 + +required: + - compatible + - reg + - interrupts + - clocks + - '#dma-cells' + - dma-channels + +allOf: + - $ref: dma-controller.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + dma-controller@fe400000 { + compatible = "amlogic,a9-dma"; + reg = <0xfe400000 0x4000>; + interrupts = ; + clocks = <&clkc 45>; + #dma-cells = <2>; + dma-channels = <28>; + }; diff --git a/Documentation/devicetree/bindings/dma/apple,admac.yaml b/Documentation/devicetree/bindings/dma/apple,admac.yaml index 6a200cbd7d02..5b4fd8348f99 100644 --- a/Documentation/devicetree/bindings/dma/apple,admac.yaml +++ b/Documentation/devicetree/bindings/dma/apple,admac.yaml @@ -10,7 +10,7 @@ description: | Apple's Audio DMA Controller (ADMAC) is used to fetch and store audio samples on SoCs from the "Apple Silicon" family. - The controller has been seen with up to 24 channels. Even-numbered channels + The controller has been seen with up to 32 channels. Even-numbered channels are TX-only, odd-numbered are RX-only. Individual channels are coupled to fixed device endpoints. @@ -33,6 +33,12 @@ properties: - apple,t8103-admac - apple,t8112-admac - const: apple,admac + - items: + - enum: + - apple,t6030-admac + - apple,t6031-admac + - const: apple,t8122-admac + - const: apple,t8122-admac reg: maxItems: 1 @@ -43,7 +49,7 @@ properties: Clients specify a single cell with channel number. dma-channels: - maximum: 24 + maximum: 32 interrupts: minItems: 4 diff --git a/Documentation/devicetree/bindings/dma/hisilicon,k3-dma-1.0.yaml b/Documentation/devicetree/bindings/dma/hisilicon,k3-dma-1.0.yaml new file mode 100644 index 000000000000..e60f2a572272 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/hisilicon,k3-dma-1.0.yaml @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/hisilicon,k3-dma-1.0.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Hisilicon K3 DMA controller + +maintainers: + - Wei Xu + +properties: + compatible: + items: + - enum: + - hisilicon,k3-dma-1.0 + - hisilicon,hisi-pcm-asp-dma-1.0 + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + maxItems: 1 + + '#dma-cells': + const: 1 + + dma-channels: + description: Number of physical DMA channels supported. + maximum: 32 + + dma-requests: + description: Number of virtual DMA request lines supported. + maximum: 32 + + dma-channel-mask: + description: Bitmask of usable DMA channels. + + dma-no-cci: + type: boolean + description: DMA is not coherent with the CCI. + + dma-type: + $ref: /schemas/types.yaml#/definitions/string + description: SoC-specific DMA type name. + enum: + - hi3660_dma + - hi6220_dma + + interrupt-names: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - '#dma-cells' + - dma-channels + - dma-requests + +allOf: + - $ref: dma-controller.yaml# + - if: + properties: + compatible: + contains: + const: hisilicon,k3-dma-1.0 + then: + required: + - clocks + +additionalProperties: false + +examples: + - | + dma-controller@fcd02000 { + compatible = "hisilicon,k3-dma-1.0"; + reg = <0xfcd02000 0x1000>; + #dma-cells = <1>; + dma-channels = <16>; + dma-requests = <27>; + interrupts = <0 12 4>; + clocks = <&pclk>; + }; diff --git a/Documentation/devicetree/bindings/dma/k3dma.txt b/Documentation/devicetree/bindings/dma/k3dma.txt deleted file mode 100644 index 10a2f15b08a3..000000000000 --- a/Documentation/devicetree/bindings/dma/k3dma.txt +++ /dev/null @@ -1,47 +0,0 @@ -* Hisilicon K3 DMA controller - -See dma.txt first - -Required properties: -- compatible: Must be one of -- "hisilicon,k3-dma-1.0" -- "hisilicon,hisi-pcm-asp-dma-1.0" -- reg: Should contain DMA registers location and length. -- interrupts: Should contain one interrupt shared by all channel -- #dma-cells: see dma.txt, should be 1, para number -- dma-channels: physical channels supported -- dma-requests: virtual channels supported, each virtual channel - have specific request line -- clocks: clock required - -Example: - -Controller: - dma0: dma@fcd02000 { - compatible = "hisilicon,k3-dma-1.0"; - reg = <0xfcd02000 0x1000>; - #dma-cells = <1>; - dma-channels = <16>; - dma-requests = <27>; - interrupts = <0 12 4>; - clocks = <&pclk>; - }; - -Client: -Use specific request line passing from dmax -For example, i2c0 read channel request line is 18, while write channel use 19 - - i2c0: i2c@fcb08000 { - compatible = "snps,designware-i2c"; - dmas = <&dma0 18 /* read channel */ - &dma0 19>; /* write channel */ - dma-names = "rx", "tx"; - }; - - i2c1: i2c@fcb09000 { - compatible = "snps,designware-i2c"; - dmas = <&dma0 20 /* read channel */ - &dma0 21>; /* write channel */ - dma-names = "rx", "tx"; - }; - diff --git a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml index bba20e88a6dc..bd5f5260613b 100644 --- a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml +++ b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml @@ -23,6 +23,7 @@ properties: - items: - enum: - mediatek,mt2712-uart-dma + - mediatek,mt6572-uart-dma - mediatek,mt6795-uart-dma - mediatek,mt8173-uart-dma - mediatek,mt8183-uart-dma diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml index 54dca623223d..dc7bdf7c4201 100644 --- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -28,7 +28,9 @@ properties: - qcom,glymur-gpi-dma - qcom,hawi-gpi-dma - qcom,kaanapali-gpi-dma + - qcom,maili-gpi-dma - qcom,milos-gpi-dma + - qcom,nord-gpi-dma - qcom,qcm2290-gpi-dma - qcom,qcs8300-gpi-dma - qcom,qdu1000-gpi-dma diff --git a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml index 04fc4a99a7cb..d3d521c927e1 100644 --- a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml +++ b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml @@ -49,6 +49,11 @@ properties: - renesas,dmac-r8a779h0 # R-Car V4M - const: renesas,rcar-gen4-dmac # R-Car Gen4 + - items: + - enum: + - renesas,dmac-r8a78000 # R-Car X5H + - const: renesas,rcar-gen5-dmac # R-Car Gen5 + reg: true interrupts: @@ -123,6 +128,7 @@ if: contains: enum: - renesas,rcar-gen4-dmac + - renesas,rcar-gen5-dmac then: properties: reg: diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml index 0a30a455b0ee..bfd6d67a98e6 100644 --- a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml +++ b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml @@ -14,6 +14,27 @@ description: allOf: - $ref: dma-controller.yaml# + - if: + properties: + compatible: + contains: + const: ultrarisc,dp1000-axi-dma + then: + properties: + reg: + maxItems: 1 + interrupts: + minItems: 8 + dma-channels: + const: 8 + snps,dma-masters: + const: 1 + snps,data-width: + const: 4 + snps,priority: + minItems: 8 + snps,block-size: + minItems: 8 properties: compatible: @@ -27,6 +48,7 @@ properties: - enum: - altr,agilex5-axi-dma - sophgo,cv1800b-axi-dma + - ultrarisc,dp1000-axi-dma - const: snps,axi-dma-1.01a reg: diff --git a/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml b/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml index d8f92838f4c9..81b83603ea79 100644 --- a/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml +++ b/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml @@ -140,8 +140,22 @@ properties: - description: LCPA (Logical Channel Parameter Address) memory. - description: LCLA (Logical Channel Link base Address) memory. + disabled-channels: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 31 + uniqueItems: true + items: + maximum: 31 + description: Physical channels that cannot be used by the DMA engine. + memcpy-channels: $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 + uniqueItems: true + items: + maximum: 63 description: Array of u32 elements indicating which channels on the DMA engine are eligible for memcpy transfers diff --git a/Documentation/devicetree/bindings/dma/ti,dra7-dma-crossbar.yaml b/Documentation/devicetree/bindings/dma/ti,dra7-dma-crossbar.yaml new file mode 100644 index 000000000000..873cedd1427d --- /dev/null +++ b/Documentation/devicetree/bindings/dma/ti,dra7-dma-crossbar.yaml @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/ti,dra7-dma-crossbar.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments DMA Crossbar (DMA request router) + +maintainers: + - Vignesh Raghavendra + - Bhargav Joshi + +description: + When requesting channel via ti,dra7-dma-crossbar, the DMA client must request + the DMA event number as crossbar ID (input to the DMA crossbar). For + ti,am335x-edma-crossbar the meaning of parameters of dmas for clients dmas = + <&edma_xbar 12 0 1>; where <12> is the DMA request number, <0> is the TC the + event should be assigned and <1> is the mux selection for in the crossbar. + When mux 0 is used the DMA channel can be requested directly from edma node. + +properties: + compatible: + enum: + - ti,dra7-dma-crossbar + - ti,am335x-edma-crossbar + + reg: + maxItems: 1 + + "#dma-cells": + minimum: 1 + maximum: 3 + + dma-requests: + minimum: 1 + maximum: 256 + + dma-masters: + maxItems: 1 + + ti,dma-safe-map: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Safe routing value for unused request lines + + ti,reserved-dma-request-ranges: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + description: + DMA request ranges which should not be used when mapping xbar input to + DMA request, they are either allocated to be used by for example the DSP + or they are used as memcpy channels in eDMA. + items: + items: + - description: starting DMA request line number + - description: number of consecutive lines to reserve + +required: + - compatible + - reg + - "#dma-cells" + - dma-requests + - dma-masters + +allOf: + - $ref: dma-router.yaml# + + - if: + properties: + compatible: + contains: + const: ti,am335x-edma-crossbar + then: + properties: + "#dma-cells": + const: 3 + + else: + properties: + "#dma-cells": + enum: [1, 2] + +unevaluatedProperties: false + +examples: + - | + dma-router@4a002b78 { + compatible = "ti,dra7-dma-crossbar"; + reg = <0x4a002b78 0xfc>; + #dma-cells = <1>; + dma-requests = <205>; + ti,dma-safe-map = <0>; + /* Protect the sDMA request ranges: 10-14 and 100-126 */ + ti,reserved-dma-request-ranges = <10 5>, <100 27>; + dma-masters = <&sdma>; + }; diff --git a/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt b/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt deleted file mode 100644 index 1f9831540c97..000000000000 --- a/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt +++ /dev/null @@ -1,68 +0,0 @@ -Texas Instruments DMA Crossbar (DMA request router) - -Required properties: -- compatible: "ti,dra7-dma-crossbar" for DRA7xx DMA crossbar - "ti,am335x-edma-crossbar" for AM335x and AM437x -- reg: Memory map for accessing module -- #dma-cells: Should be set to match with the DMA controller's dma-cells - for ti,dra7-dma-crossbar and <3> for ti,am335x-edma-crossbar. -- dma-requests: Number of DMA requests the crossbar can receive -- dma-masters: phandle pointing to the DMA controller - -The DMA controller node need to have the following poroperties: -- dma-requests: Number of DMA requests the controller can handle - -Optional properties: -- ti,dma-safe-map: Safe routing value for unused request lines -- ti,reserved-dma-request-ranges: DMA request ranges which should not be used - when mapping xbar input to DMA request, they are either - allocated to be used by for example the DSP or they are used as - memcpy channels in eDMA. - -Notes: -When requesting channel via ti,dra7-dma-crossbar, the DMA client must request -the DMA event number as crossbar ID (input to the DMA crossbar). - -For ti,am335x-edma-crossbar: the meaning of parameters of dmas for clients: -dmas = <&edma_xbar 12 0 1>; where <12> is the DMA request number, <0> is the TC -the event should be assigned and <1> is the mux selection for in the crossbar. -When mux 0 is used the DMA channel can be requested directly from edma node. - -Example: - -/* DMA controller */ -sdma: dma-controller@4a056000 { - compatible = "ti,omap4430-sdma"; - reg = <0x4a056000 0x1000>; - interrupts = , - , - , - ; - #dma-cells = <1>; - dma-channels = <32>; - dma-requests = <127>; -}; - -/* DMA crossbar */ -sdma_xbar: dma-router@4a002b78 { - compatible = "ti,dra7-dma-crossbar"; - reg = <0x4a002b78 0xfc>; - #dma-cells = <1>; - dma-requests = <205>; - ti,dma-safe-map = <0>; - /* Protect the sDMA request ranges: 10-14 and 100-126 */ - ti,reserved-dma-request-ranges = <10 5>, <100 27>; - dma-masters = <&sdma>; -}; - -/* DMA client */ -uart1: serial@4806a000 { - compatible = "ti,omap4-uart"; - reg = <0x4806a000 0x100>; - interrupts-extended = <&gic GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>; - ti,hwmods = "uart1"; - clock-frequency = <48000000>; - /* Requesting crossbar input 49 and 50 */ - dmas = <&sdma_xbar 49>, <&sdma_xbar 50>; - dma-names = "tx", "rx"; -}; diff --git a/Documentation/driver-api/dmaengine/provider.rst b/Documentation/driver-api/dmaengine/provider.rst index f4ed98f701c9..be47852e5a0f 100644 --- a/Documentation/driver-api/dmaengine/provider.rst +++ b/Documentation/driver-api/dmaengine/provider.rst @@ -321,7 +321,7 @@ to use. - DESC_METADATA_CLIENT The metadata buffer is allocated/provided by the client driver and it is - attached (via the dmaengine_desc_attach_metadata() helper to the descriptor. + attached (via the dmaengine_desc_attach_metadata() helper) to the descriptor. From the DMA driver the following is expected for this mode: @@ -413,9 +413,11 @@ supported. - You'll also need to set following fields in this structure: - - flags: - TODO: Can it be modified by the driver itself, or - should it be always the flags passed in the arguments + - flags: this field belongs to the client. The driver stores the + flags passed in the arguments and must not modify them. It reads + the DMA_PREP_* bits while preparing the descriptor, and must only + test DMA_CTRL_ACK and DMA_CTRL_REUSE, which the client may change + after the descriptor has been prepared. - tx_submit: A pointer to a function you have to implement, that is supposed to push the current transaction descriptor to a @@ -555,6 +557,11 @@ dma_cookie_t - Not really relevant any more since the introduction of ``virt-dma`` that abstracts it away. +- dmaengine drivers need to ensure buffers are flushed before invoking + vchan_cookie_complete() or dma_cookie_complete(). This would imply a call + to unmapping buffers or any such architecture calls to ensure caller seeing + buffers are flushed is done before invoking these calls. + dma_vec - A small structure that contains a DMA address and length. diff --git a/MAINTAINERS b/MAINTAINERS index 139c45547ad4..81fcd0bb9dbe 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1391,6 +1391,13 @@ F: Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml F: drivers/perf/amlogic/ F: include/soc/amlogic/ +AMLOGIC DMA DRIVER +M: Xianwei Zhao +L: linux-amlogic@lists.infradead.org +S: Maintained +F: Documentation/devicetree/bindings/dma/amlogic,a9-dma.yaml +F: drivers/dma/amlogic-dma.c + AMLOGIC ISP DRIVER M: Keke Li L: linux-media@vger.kernel.org @@ -27622,7 +27629,7 @@ TEXAS INSTRUMENTS DMA DRIVERS M: Vignesh Raghavendra L: dmaengine@vger.kernel.org S: Maintained -F: Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt +F: Documentation/devicetree/bindings/dma/ti,dra7-dma-crossbar.yaml F: Documentation/devicetree/bindings/dma/ti-edma.txt F: Documentation/devicetree/bindings/dma/ti/ F: drivers/dma/ti/ diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index ae6a682c9f76..01f96a8257e5 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -85,6 +85,16 @@ config AMCC_PPC440SPE_ADMA help Enable support for the AMCC PPC440SPe RAID engines. +config AMLOGIC_DMA + tristate "Amlogic general DMA support" + depends on ARCH_MESON || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + select REGMAP_MMIO + help + Enable support for the Amlogic general DMA engines. THis DMA + controller is used some Amlogic SoCs, such as A9. + config APPLE_ADMAC tristate "Apple ADMAC support" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 14aa086629d5..f62d12b08e15 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -16,6 +16,7 @@ obj-$(CONFIG_DMATEST) += dmatest.o obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ +obj-$(CONFIG_AMLOGIC_DMA) += amlogic-dma.o obj-$(CONFIG_APPLE_ADMAC) += apple-admac.o obj-$(CONFIG_ARM_DMA350) += arm-dma350.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o diff --git a/drivers/dma/altera-msgdma.c b/drivers/dma/altera-msgdma.c index e23e5b441a24..412f4bce1d4a 100644 --- a/drivers/dma/altera-msgdma.c +++ b/drivers/dma/altera-msgdma.c @@ -847,11 +847,6 @@ static int msgdma_probe(struct platform_device *pdev) if (mdev->irq < 0) return -ENXIO; - ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler, - 0, dev_name(&pdev->dev), mdev); - if (ret) - return ret; - tasklet_setup(&mdev->irq_tasklet, msgdma_tasklet); dma_cookie_init(&mdev->dmachan); @@ -863,6 +858,11 @@ static int msgdma_probe(struct platform_device *pdev) INIT_LIST_HEAD(&mdev->done_list); INIT_LIST_HEAD(&mdev->free_list); + ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler, + 0, dev_name(&pdev->dev), mdev); + if (ret) + return ret; + dma_dev = &mdev->dmadev; /* Set DMA capabilities */ diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index 5e88fd44812d..543c2eb8cfa8 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -2500,8 +2500,8 @@ DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs); static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) { /* Expose a simple debugfs interface to view all clocks */ - debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO, - NULL, pl08x, &pl08x_debugfs_fops); + debugfs_create_file("stats", 0444, pl08x->memcpy.dbg_dev_root, + pl08x, &pl08x_debugfs_fops); } #else diff --git a/drivers/dma/amlogic-dma.c b/drivers/dma/amlogic-dma.c new file mode 100644 index 000000000000..684121d13fca --- /dev/null +++ b/drivers/dma/amlogic-dma.c @@ -0,0 +1,737 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) +/* + * Copyright (C) 2026 Amlogic, Inc. All rights reserved + * Author: Xianwei Zhao + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "virt-dma.h" + +#define RCH_REG_BASE 0x0 +#define WCH_REG_BASE 0x2000 +/* + * Each rch (read from memory) REG offset Rch_offset 0x0 each channel total 0x40 + * rch addr = DMA_base + Rch_offset+ chan_id * 0x40 + reg_offset + */ +#define RCH_READY 0x0 +#define RCH_STATUS 0x4 +#define RCH_CFG 0x8 +#define CFG_CLEAR BIT(25) +#define CFG_PAUSE BIT(26) +#define CFG_ENABLE BIT(27) +#define CFG_DONE BIT(28) +#define RCH_ADDR 0xc +#define RCH_LEN 0x10 +#define RCH_RD_LEN 0x14 +#define RCH_PRT 0x18 +#define RCH_SYCN_STAT 0x1c +#define RCH_ADDR_LOW 0x20 +#define RCH_ADDR_HIGH 0x24 +/* if work on 64, it work with RCH_PRT */ +#define RCH_PTR_HIGH 0x28 + +/* + * Each wch (write to memory) REG offset Wch_offset 0x2000 each channel total 0x40 + * wch addr = DMA_base + Wch_offset+ chan_id * 0x40 + reg_offset + */ +#define WCH_READY 0x0 +#define WCH_TOTAL_LEN 0x4 +#define WCH_CFG 0x8 +#define WCH_ADDR 0xc +#define WCH_LEN 0x10 +#define WCH_RD_LEN 0x14 +#define WCH_PRT 0x18 +#define WCH_CMD_CNT 0x1c +#define WCH_ADDR_LOW 0x20 +#define WCH_ADDR_HIGH 0x24 +/* if work on 64, it work with RCH_PRT */ +#define WCH_PTR_HIGH 0x28 + +/* DMA controller reg */ +#define RCH_INT_MASK 0x1000 +#define WCH_INT_MASK 0x1004 +#define CLEAR_W_BATCH 0x1014 +#define CLEAR_RCH 0x1024 +#define CLEAR_WCH 0x1028 +#define RCH_ACTIVE 0x1038 +#define WCH_ACTIVE 0x103c +#define RCH_DONE 0x104c +#define WCH_DONE 0x1050 +#define RCH_ERR 0x1060 +#define RCH_LEN_ERR 0x1064 +#define WCH_ERR 0x1068 +#define DMA_BATCH_END 0x1078 +#define WCH_EOC_DONE 0x1088 +#define WDMA_RESP_ERR 0x1098 +#define UPT_PKT_SYNC 0x10a8 +#define RCHN_CFG 0x10ac +#define WCHN_CFG 0x10b0 +#define MEM_PD_CFG 0x10b4 +#define MEM_BUS_CFG 0x10b8 +#define DMA_GMV_CFG 0x10bc +#define DMA_GMR_CFG 0x10c0 + +#define MAX_CHAN_ID 32 +#define SG_MAX_LEN GENMASK(26, 2) +#define INT_MASK 0xffffffff + +struct aml_dma_sg_link { +#define LINK_LEN GENMASK(26, 0) +#define LINK_IRQ BIT(27) +#define LINK_EOC BIT(28) +#define LINK_LOOP BIT(29) +#define LINK_ERR BIT(30) +#define LINK_OWNER BIT(31) + __le32 ctl; + __le32 addr_low; + __le32 addr_high; + __le32 revered; +} __packed; + +/* 1 page for link 256*16 */ +#define DMA_MAX_LINK 256 +/* sizeof(struct aml_dma_sg_link) */ +#define DMA_LINK_SIZE 16 +#define DMA_LINK_MAX_SIZE (DMA_LINK_SIZE * DMA_MAX_LINK) + +struct aml_dma_desc { + struct virt_dma_desc vd; + struct aml_dma_sg_link *sg_link; + struct dma_device *dma_device; + dma_addr_t sg_link_phys; + size_t sg_link_size; + u32 data_len; + struct work_struct free_work; +}; + +struct aml_dma_chan { + struct virt_dma_chan vchan; + struct aml_dma_dev *aml_dma; + struct aml_dma_desc *cur_desc; + enum dma_status pre_status; + enum dma_status status; + enum dma_transfer_direction direction; + int chan_id; + /* reg_base (direction + chan_id) */ + int reg_offs; + /* When there are multiple consecutive transmission errors, this channel halt */ + int err_num; +}; + +struct aml_dma_dev { + struct dma_device dma_device; + void __iomem *base; + struct regmap *regmap; + struct clk *clk; + int irq; + struct platform_device *pdev; + struct aml_dma_chan *aml_rch[MAX_CHAN_ID]; + struct aml_dma_chan *aml_wch[MAX_CHAN_ID]; + struct workqueue_struct *free_wq; + unsigned int chan_nr; + unsigned int chan_used; + struct aml_dma_chan aml_chans[]__counted_by(chan_nr); +}; + +static inline struct aml_dma_chan *to_aml_dma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct aml_dma_chan, vchan.chan); +} + +static inline struct aml_dma_desc *to_aml_dma_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct aml_dma_desc, vd); +} + +static void aml_dma_free_desc_work(struct work_struct *work) +{ + struct aml_dma_desc *aml_desc = + container_of(work, struct aml_dma_desc, free_work); + + dma_free_coherent(aml_desc->dma_device->dev, + aml_desc->sg_link_size, + aml_desc->sg_link, + aml_desc->sg_link_phys); + kfree(aml_desc); +} + +static void aml_dma_free_desc(struct virt_dma_desc *vd) +{ + struct aml_dma_desc *aml_desc = to_aml_dma_desc(vd); + struct aml_dma_dev *aml_dev = container_of(aml_desc->dma_device, + struct aml_dma_dev, dma_device); + + queue_work(aml_dev->free_wq, &aml_desc->free_work); +} + +static int aml_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct aml_dma_chan *aml_chan = to_aml_dma_chan(chan); + struct aml_dma_dev *aml_dma = aml_chan->aml_dma; + + /* offset is the same RCH_CFG and WCH_CFG */ + regmap_set_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_CLEAR); + regmap_clear_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_PAUSE); + regmap_clear_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_CLEAR); + + aml_chan->status = DMA_COMPLETE; + aml_chan->cur_desc = NULL; + aml_chan->err_num = 0; + + return 0; +} + +static void aml_dma_free_chan_resources(struct dma_chan *chan) +{ + struct aml_dma_chan *aml_chan = to_aml_dma_chan(chan); + struct aml_dma_dev *aml_dma = aml_chan->aml_dma; + struct virt_dma_desc *cur_vd = NULL; + unsigned long flags; + + spin_lock_irqsave(&aml_chan->vchan.lock, flags); + regmap_set_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_PAUSE); + regmap_set_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_CLEAR); + if (aml_chan->cur_desc) + cur_vd = &aml_chan->cur_desc->vd; + aml_chan->cur_desc = NULL; + spin_unlock_irqrestore(&aml_chan->vchan.lock, flags); + if (cur_vd) + aml_dma_free_desc(cur_vd); + + vchan_free_chan_resources(&aml_chan->vchan); +} + +/* DMA transfer state update how many data reside it */ +static enum dma_status aml_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct aml_dma_chan *aml_chan = to_aml_dma_chan(chan); + struct aml_dma_dev *aml_dma = aml_chan->aml_dma; + struct aml_dma_desc *aml_desc = NULL; + struct virt_dma_desc *vd; + u32 residue = 0, done = 0; + unsigned long flags; + enum dma_status ret; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + if (aml_chan->status == DMA_PAUSED) + ret = DMA_PAUSED; + if (!txstate) + return ret; + + spin_lock_irqsave(&aml_chan->vchan.lock, flags); + vd = vchan_find_desc(&aml_chan->vchan, cookie); + if (vd) { + aml_desc = to_aml_dma_desc(vd); + residue = aml_desc->data_len; + } else if (aml_chan->cur_desc && aml_chan->cur_desc->vd.tx.cookie == cookie) { + aml_desc = aml_chan->cur_desc; + regmap_read(aml_dma->regmap, aml_chan->reg_offs + RCH_RD_LEN, &done); + residue = aml_desc->data_len - done; + } else { + dev_err(aml_dma->dma_device.dev, "cookie error\n"); + } + spin_unlock_irqrestore(&aml_chan->vchan.lock, flags); + dma_set_residue(txstate, residue); + + return ret; +} + +static struct dma_async_tx_descriptor *aml_dma_prep_slave_sg + (struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct aml_dma_chan *aml_chan = to_aml_dma_chan(chan); + struct aml_dma_dev *aml_dma = aml_chan->aml_dma; + struct aml_dma_desc *aml_desc = NULL; + struct aml_dma_sg_link *sg_link = NULL; + struct scatterlist *sg = NULL; + u64 paddr; + u32 link_count, avail; + u32 i; + + if (aml_chan->direction != direction) { + dev_err(aml_dma->dma_device.dev, "direction not support\n"); + return NULL; + } + + link_count = sg_nents_for_dma(sgl, sg_len, SG_MAX_LEN); + if (link_count == 0) + return NULL; + + aml_desc = kzalloc_obj(*aml_desc, GFP_NOWAIT); + if (!aml_desc) + return NULL; + + INIT_WORK(&aml_desc->free_work, aml_dma_free_desc_work); + aml_desc->sg_link_size = link_count * sizeof(*sg_link); + aml_desc->sg_link = dma_alloc_coherent(aml_dma->dma_device.dev, aml_desc->sg_link_size, + &aml_desc->sg_link_phys, GFP_NOWAIT); + if (!aml_desc->sg_link) { + kfree(aml_desc); + return NULL; + } + aml_desc->dma_device = &aml_dma->dma_device; + + sg_link = aml_desc->sg_link; + for_each_sg(sgl, sg, sg_len, i) { + avail = sg_dma_len(sg); + if (avail == 0) + continue; + paddr = sg->dma_address; + while (avail > SG_MAX_LEN) { + /* set dma address and len to sglink*/ + sg_link->addr_low = cpu_to_le32(lower_32_bits(paddr)); + sg_link->addr_high = cpu_to_le32(upper_32_bits(paddr)); + sg_link->ctl = cpu_to_le32(FIELD_PREP(LINK_LEN, SG_MAX_LEN)); + paddr = paddr + SG_MAX_LEN; + avail = avail - SG_MAX_LEN; + sg_link++; + } + /* set dma address and len to sglink*/ + sg_link->addr_low = cpu_to_le32(lower_32_bits(paddr)); + sg_link->addr_high = cpu_to_le32(upper_32_bits(paddr)); + sg_link->ctl = cpu_to_le32(FIELD_PREP(LINK_LEN, avail)); + + aml_desc->data_len += sg_dma_len(sg); + sg_link++; + } + + /* the last sg set eoc flag */ + sg_link--; + sg_link->ctl |= LINK_EOC; + + return vchan_tx_prep(&aml_chan->vchan, &aml_desc->vd, flags); +} + +static int aml_dma_chan_pause(struct dma_chan *chan) +{ + struct aml_dma_chan *aml_chan = to_aml_dma_chan(chan); + struct aml_dma_dev *aml_dma = aml_chan->aml_dma; + unsigned long flags; + + spin_lock_irqsave(&aml_chan->vchan.lock, flags); + regmap_set_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_PAUSE); + aml_chan->pre_status = aml_chan->status; + aml_chan->status = DMA_PAUSED; + spin_unlock_irqrestore(&aml_chan->vchan.lock, flags); + + return 0; +} + +static int aml_dma_chan_resume(struct dma_chan *chan) +{ + struct aml_dma_chan *aml_chan = to_aml_dma_chan(chan); + struct aml_dma_dev *aml_dma = aml_chan->aml_dma; + unsigned long flags; + + spin_lock_irqsave(&aml_chan->vchan.lock, flags); + regmap_clear_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_PAUSE); + aml_chan->status = aml_chan->pre_status; + spin_unlock_irqrestore(&aml_chan->vchan.lock, flags); + + return 0; +} + +static int aml_dma_terminate_all(struct dma_chan *chan) +{ + struct aml_dma_chan *aml_chan = to_aml_dma_chan(chan); + struct aml_dma_dev *aml_dma = aml_chan->aml_dma; + int chan_id = aml_chan->chan_id; + struct virt_dma_desc *cur_vd; + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&aml_chan->vchan.lock, flags); + regmap_set_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_PAUSE); + regmap_set_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_CLEAR); + + if (aml_chan->direction == DMA_MEM_TO_DEV) + regmap_set_bits(aml_dma->regmap, RCH_INT_MASK, BIT(chan_id)); + else if (aml_chan->direction == DMA_DEV_TO_MEM) + regmap_set_bits(aml_dma->regmap, WCH_INT_MASK, BIT(chan_id)); + + regmap_clear_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_PAUSE); + regmap_clear_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_CLEAR); + + vchan_get_all_descriptors(&aml_chan->vchan, &head); + cur_vd = &aml_chan->cur_desc->vd; + aml_chan->cur_desc = NULL; + spin_unlock_irqrestore(&aml_chan->vchan.lock, flags); + if (cur_vd) + aml_dma_free_desc(cur_vd); + + vchan_dma_desc_free_list(&aml_chan->vchan, &head); + + aml_chan->status = DMA_COMPLETE; + aml_chan->err_num = 0; + + return 0; +} + +static void aml_dma_start(struct aml_dma_chan *aml_chan) +{ + struct virt_dma_desc *vd = vchan_next_desc(&aml_chan->vchan); + struct aml_dma_dev *aml_dma = aml_chan->aml_dma; + struct aml_dma_desc *aml_desc = NULL; + int chan_id = aml_chan->chan_id; + + if (aml_chan->status == DMA_ERROR) { + if (aml_chan->err_num > 5) { + dev_err(aml_dma->dma_device.dev, "hw error\n"); + return; + } + regmap_set_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_PAUSE); + regmap_set_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_CLEAR); + aml_chan->err_num++; + regmap_clear_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_PAUSE); + regmap_clear_bits(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, CFG_CLEAR); + aml_chan->status = DMA_COMPLETE; + } else { + aml_chan->err_num = 0; + } + + if (!vd) + return; + + if (aml_chan->status != DMA_COMPLETE) + return; + + list_del(&vd->node); + aml_desc = to_aml_dma_desc(vd); + aml_chan->cur_desc = aml_desc; + + if (aml_chan->direction == DMA_MEM_TO_DEV) { + regmap_write(aml_dma->regmap, aml_chan->reg_offs + RCH_ADDR, + lower_32_bits(aml_desc->sg_link_phys)); + regmap_write(aml_dma->regmap, aml_chan->reg_offs + RCH_ADDR_HIGH, + upper_32_bits(aml_desc->sg_link_phys)); + regmap_write(aml_dma->regmap, aml_chan->reg_offs + RCH_LEN, aml_desc->data_len); + regmap_clear_bits(aml_dma->regmap, RCH_INT_MASK, BIT(chan_id)); + /* for rch (tx) need set cfg 0 to trigger start */ + regmap_write(aml_dma->regmap, aml_chan->reg_offs + RCH_CFG, 0); + } else if (aml_chan->direction == DMA_DEV_TO_MEM) { + regmap_write(aml_dma->regmap, aml_chan->reg_offs + WCH_ADDR, + lower_32_bits(aml_desc->sg_link_phys)); + regmap_write(aml_dma->regmap, aml_chan->reg_offs + WCH_ADDR_HIGH, + upper_32_bits(aml_desc->sg_link_phys)); + regmap_write(aml_dma->regmap, aml_chan->reg_offs + WCH_LEN, aml_desc->data_len); + regmap_clear_bits(aml_dma->regmap, WCH_INT_MASK, BIT(chan_id)); + } +} + +static void aml_dma_issue_pending(struct dma_chan *chan) +{ + struct aml_dma_chan *aml_chan = to_aml_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&aml_chan->vchan.lock, flags); + if (vchan_issue_pending(&aml_chan->vchan) && !aml_chan->cur_desc) + aml_dma_start(aml_chan); + spin_unlock_irqrestore(&aml_chan->vchan.lock, flags); +} + +static irqreturn_t aml_dma_interrupt_handler(int irq, void *dev_id) +{ + struct aml_dma_dev *aml_dma = dev_id; + struct aml_dma_chan *aml_chan; + struct aml_dma_desc *aml_desc; + u32 done, eoc_done, err, err_l, end; + u32 cpl_data; + int i = 0; + + /* deal with rch normal complete and error */ + regmap_read(aml_dma->regmap, RCH_DONE, &done); + regmap_read(aml_dma->regmap, RCH_ERR, &err); + regmap_read(aml_dma->regmap, RCH_LEN_ERR, &err_l); + err = err | err_l; + + done = done | err; + + while (done) { + i = ffs(done) - 1; + regmap_write(aml_dma->regmap, CLEAR_RCH, BIT(i)); + done &= ~BIT(i); + aml_chan = aml_dma->aml_rch[i]; + if (!aml_chan) { + dev_err(aml_dma->dma_device.dev, "idx %d rch not initialized\n", i); + continue; + } + spin_lock(&aml_chan->vchan.lock); + aml_chan->status = (err & BIT(i)) ? DMA_ERROR : DMA_COMPLETE; + aml_desc = aml_chan->cur_desc; + if (!aml_desc) { + spin_unlock(&aml_chan->vchan.lock); + continue; + } + if (aml_chan->status == DMA_ERROR) { + aml_desc->vd.tx_result.result = DMA_TRANS_READ_FAILED; + regmap_read(aml_dma->regmap, aml_chan->reg_offs + RCH_RD_LEN, &cpl_data); + aml_desc->vd.tx_result.residue = aml_desc->data_len - cpl_data; + } + vchan_cookie_complete(&aml_desc->vd); + aml_chan->cur_desc = NULL; + aml_dma_start(aml_chan); + spin_unlock(&aml_chan->vchan.lock); + } + + /* deal with wch normal complete and error */ + regmap_read(aml_dma->regmap, DMA_BATCH_END, &end); + if (end) + regmap_write(aml_dma->regmap, CLEAR_W_BATCH, end); + + regmap_read(aml_dma->regmap, WCH_DONE, &done); + regmap_read(aml_dma->regmap, WCH_EOC_DONE, &eoc_done); + done = done | eoc_done; + + regmap_read(aml_dma->regmap, WCH_ERR, &err); + regmap_read(aml_dma->regmap, WDMA_RESP_ERR, &err_l); + err = err | err_l; + + done = done | err; + i = 0; + while (done) { + i = ffs(done) - 1; + done &= ~BIT(i); + regmap_write(aml_dma->regmap, CLEAR_WCH, BIT(i)); + aml_chan = aml_dma->aml_wch[i]; + if (!aml_chan) { + dev_err(aml_dma->dma_device.dev, "idx %d wch not initialized\n", i); + continue; + } + spin_lock(&aml_chan->vchan.lock); + aml_chan->status = (err & BIT(i)) ? DMA_ERROR : DMA_COMPLETE; + aml_desc = aml_chan->cur_desc; + if (!aml_desc) { + spin_unlock(&aml_chan->vchan.lock); + continue; + } + if (aml_chan->status == DMA_ERROR) { + aml_desc->vd.tx_result.result = DMA_TRANS_WRITE_FAILED; + regmap_read(aml_dma->regmap, aml_chan->reg_offs + RCH_RD_LEN, &cpl_data); + aml_desc->vd.tx_result.residue = aml_desc->data_len - cpl_data; + } + vchan_cookie_complete(&aml_desc->vd); + aml_chan->cur_desc = NULL; + aml_dma_start(aml_chan); + spin_unlock(&aml_chan->vchan.lock); + } + + return IRQ_HANDLED; +} + +static struct dma_chan *aml_of_dma_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) +{ + struct aml_dma_dev *aml_dma = (struct aml_dma_dev *)ofdma->of_dma_data; + struct aml_dma_chan *aml_chan = NULL; + u32 type; + u32 phy_chan_id; + + if (dma_spec->args_count != 2) + return NULL; + + type = dma_spec->args[0]; + phy_chan_id = dma_spec->args[1]; + + if (phy_chan_id >= MAX_CHAN_ID) + return NULL; + + if (type == AMLOGIC_A9_DMA_TX) { + aml_chan = aml_dma->aml_rch[phy_chan_id]; + if (!aml_chan) { + if (aml_dma->chan_used >= aml_dma->chan_nr) { + dev_err(aml_dma->dma_device.dev, "some dma clients err used\n"); + return NULL; + } + aml_chan = &aml_dma->aml_chans[aml_dma->chan_used]; + aml_dma->chan_used++; + aml_chan->direction = DMA_MEM_TO_DEV; + aml_chan->chan_id = phy_chan_id; + aml_chan->reg_offs = RCH_REG_BASE + 0x40 * aml_chan->chan_id; + aml_dma->aml_rch[phy_chan_id] = aml_chan; + } + } else if (type == AMLOGIC_A9_DMA_RX) { + aml_chan = aml_dma->aml_wch[phy_chan_id]; + if (!aml_chan) { + if (aml_dma->chan_used >= aml_dma->chan_nr) { + dev_err(aml_dma->dma_device.dev, "some dma clients err used\n"); + return NULL; + } + aml_chan = &aml_dma->aml_chans[aml_dma->chan_used]; + aml_dma->chan_used++; + aml_chan->direction = DMA_DEV_TO_MEM; + aml_chan->chan_id = phy_chan_id; + aml_chan->reg_offs = WCH_REG_BASE + 0x40 * aml_chan->chan_id; + aml_dma->aml_wch[phy_chan_id] = aml_chan; + } + } else { + dev_err(aml_dma->dma_device.dev, "type %d not supported\n", type); + return NULL; + } + + return dma_get_slave_channel(&aml_chan->vchan.chan); +} + +static int aml_dma_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct dma_device *dma_dev; + struct aml_dma_dev *aml_dma; + int ret, i, len; + u32 chan_nr; + + const struct regmap_config aml_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = 0x3000, + }; + + ret = of_property_read_u32(np, "dma-channels", &chan_nr); + if (ret) + return dev_err_probe(&pdev->dev, ret, "failed to read dma-channels\n"); + if (chan_nr > (MAX_CHAN_ID * 2)) + return dev_err_probe(&pdev->dev, -EINVAL, "dma-channels unusual\n"); + + len = sizeof(struct aml_dma_dev) + sizeof(struct aml_dma_chan) * chan_nr; + aml_dma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + if (!aml_dma) + return -ENOMEM; + + aml_dma->free_wq = alloc_ordered_workqueue("aml_dma_free", 0); + if (!aml_dma->free_wq) + return -ENOMEM; + + aml_dma->chan_nr = chan_nr; + + aml_dma->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(aml_dma->base)) + return PTR_ERR(aml_dma->base); + + aml_dma->regmap = devm_regmap_init_mmio(&pdev->dev, aml_dma->base, + &aml_regmap_config); + if (IS_ERR(aml_dma->regmap)) + return PTR_ERR(aml_dma->regmap); + + aml_dma->clk = devm_clk_get_enabled(&pdev->dev, NULL); + if (IS_ERR(aml_dma->clk)) + return PTR_ERR(aml_dma->clk); + + aml_dma->irq = platform_get_irq(pdev, 0); + + aml_dma->pdev = pdev; + aml_dma->dma_device.dev = &pdev->dev; + + dma_dev = &aml_dma->dma_device; + INIT_LIST_HEAD(&dma_dev->channels); + + /* Initialize channel parameters */ + for (i = 0; i < chan_nr; i++) { + struct aml_dma_chan *aml_chan = &aml_dma->aml_chans[i]; + + aml_chan->aml_dma = aml_dma; + aml_chan->vchan.desc_free = aml_dma_free_desc; + vchan_init(&aml_chan->vchan, &aml_dma->dma_device); + } + aml_dma->chan_used = 0; + + dma_set_max_seg_size(dma_dev->dev, SG_MAX_LEN); + dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); + dma_dev->device_alloc_chan_resources = aml_dma_alloc_chan_resources; + dma_dev->device_free_chan_resources = aml_dma_free_chan_resources; + dma_dev->device_tx_status = aml_dma_tx_status; + dma_dev->device_prep_slave_sg = aml_dma_prep_slave_sg; + dma_dev->device_pause = aml_dma_chan_pause; + dma_dev->device_resume = aml_dma_chan_resume; + dma_dev->device_terminate_all = aml_dma_terminate_all; + dma_dev->device_issue_pending = aml_dma_issue_pending; + /* PIO 4 bytes and I2C 1 byte */ + dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_1_BYTE); + dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + regmap_write(aml_dma->regmap, RCH_INT_MASK, INT_MASK); + regmap_write(aml_dma->regmap, WCH_INT_MASK, INT_MASK); + + ret = devm_request_irq(&pdev->dev, aml_dma->irq, aml_dma_interrupt_handler, + 0, dev_name(&pdev->dev), aml_dma); + if (ret) + return dev_err_probe(&pdev->dev, ret, "failed to register irq\n"); + + ret = dmaenginem_async_device_register(dma_dev); + if (ret) + return dev_err_probe(&pdev->dev, ret, "failed to register dmaenginem\n"); + + ret = of_dma_controller_register(np, aml_of_dma_xlate, aml_dma); + if (ret) + return dev_err_probe(&pdev->dev, ret, "failed to register xlate\n"); + + platform_set_drvdata(pdev, aml_dma); + + return 0; +} + +static void aml_dma_remove(struct platform_device *pdev) +{ + struct aml_dma_dev *aml_dma = platform_get_drvdata(pdev); + struct aml_dma_chan *aml_chan = NULL; + int i; + + of_dma_controller_free((&pdev->dev)->of_node); + + regmap_write(aml_dma->regmap, RCH_INT_MASK, INT_MASK); + regmap_write(aml_dma->regmap, WCH_INT_MASK, INT_MASK); + + for (i = 0; i < MAX_CHAN_ID; i++) { + aml_chan = aml_dma->aml_rch[i]; + if (aml_chan) + tasklet_kill(&aml_chan->vchan.task); + aml_chan = aml_dma->aml_wch[i]; + if (aml_chan) + tasklet_kill(&aml_chan->vchan.task); + } + destroy_workqueue(aml_dma->free_wq); +} + +static const struct of_device_id aml_dma_ids[] = { + { .compatible = "amlogic,a9-dma", }, + { } +}; +MODULE_DEVICE_TABLE(of, aml_dma_ids); + +static struct platform_driver aml_dma_driver = { + .probe = aml_dma_probe, + .remove = aml_dma_remove, + .driver = { + .name = "aml-dma", + .of_match_table = aml_dma_ids, + }, +}; + +module_platform_driver(aml_dma_driver); + +MODULE_DESCRIPTION("GENERAL DMA driver for Amlogic"); +MODULE_AUTHOR("Xianwei Zhao "); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index 14a5ee14a481..9a1c79c12268 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -39,10 +39,14 @@ #define FLAG_DESC_NOTIFY BIT(16) +#define T8122_UNK_28_VAL 0x200000 + #define REG_TX_START 0x0000 #define REG_TX_STOP 0x0004 #define REG_RX_START 0x0008 #define REG_RX_STOP 0x000c +#define REG_UNK_28 0x0028 +#define REG_UNK_2C 0x002c #define REG_IMPRINT 0x0090 #define REG_TX_SRAM_SIZE 0x0094 #define REG_RX_SRAM_SIZE 0x0098 @@ -85,6 +89,10 @@ struct admac_data; struct admac_tx; +struct admac_hw { + bool set_unk28; +}; + struct admac_chan { unsigned int no; struct admac_data *host; @@ -127,6 +135,7 @@ struct admac_data { struct mutex cache_alloc_lock; struct admac_sram txcache, rxcache; + const struct admac_hw *hw; int irq; int irq_index; int nchannels; @@ -747,6 +756,11 @@ static int admac_device_config(struct dma_chan *chan, u32 bus_width = readl_relaxed(ad->base + REG_BUS_WIDTH(adchan->no)) & ~(BUS_WIDTH_WORD_SIZE | BUS_WIDTH_FRAME_SIZE); + if (ad->hw->set_unk28) { + writel_relaxed(T8122_UNK_28_VAL, ad->base + REG_UNK_28); + writel_relaxed(T8122_UNK_28_VAL, ad->base + REG_UNK_2C); + } + switch (is_tx ? config->dst_addr_width : config->src_addr_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: wordsize = 1; @@ -800,11 +814,18 @@ static int admac_device_config(struct dma_chan *chan, return 0; } +static const struct admac_hw admac_base_hw = {}; + +static const struct admac_hw admac_t8122_hw = { + .set_unk28 = true, +}; + static int admac_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct admac_data *ad; struct dma_device *dma; + const struct admac_hw *hw; int nchannels; int err, irq, i; @@ -822,6 +843,8 @@ static int admac_probe(struct platform_device *pdev) ad->dev = &pdev->dev; ad->nchannels = nchannels; mutex_init(&ad->cache_alloc_lock); + hw = of_device_get_match_data(&pdev->dev); + ad->hw = hw ? hw : &admac_base_hw; /* * The controller has 4 IRQ outputs. Try them all until @@ -936,6 +959,7 @@ static void admac_remove(struct platform_device *pdev) } static const struct of_device_id admac_of_match[] = { + { .compatible = "apple,t8122-admac", .data = &admac_t8122_hw }, { .compatible = "apple,t8103-admac", }, { .compatible = "apple,admac", }, { } diff --git a/drivers/dma/arm-dma350.c b/drivers/dma/arm-dma350.c index 09403aca8bb0..52f5242ea332 100644 --- a/drivers/dma/arm-dma350.c +++ b/drivers/dma/arm-dma350.c @@ -512,7 +512,7 @@ static int d350_alloc_chan_resources(struct dma_chan *chan) { struct d350_chan *dch = to_d350_chan(chan); int ret = request_irq(dch->irq, d350_irq, IRQF_SHARED, - dev_name(&dch->vc.chan.dev->device), dch); + vchan_chan_name(&dch->vc), dch); if (!ret) writel_relaxed(CH_INTREN_DONE | CH_INTREN_ERR, dch->base + CH_INTREN); diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index e5b30a57c477..3fedc907db63 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -244,7 +244,7 @@ struct at_desc { /* Memset temporary buffer */ bool memset_buffer; dma_addr_t memset_paddr; - int *memset_vaddr; + u32 *memset_vaddr; struct atdma_sg sg[] __counted_by(sglen); }; @@ -381,23 +381,18 @@ static inline struct at_dma *to_at_dma(struct dma_device *ddev) /*-- Helper functions ------------------------------------------------*/ -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - #if defined(VERBOSE_DEBUG) static void vdbg_dump_regs(struct at_dma_chan *atchan) { struct at_dma *atdma = to_at_dma(atchan->vc.chan.device); - dev_err(chan2dev(&atchan->vc.chan), + dev_err(vchan_chan_dev(&atchan->vc), " channel %d : imr = 0x%x, chsr = 0x%x\n", atchan->vc.chan.chan_id, dma_readl(atdma, EBCIMR), dma_readl(atdma, CHSR)); - dev_err(chan2dev(&atchan->vc.chan), + dev_err(vchan_chan_dev(&atchan->vc), " channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n", channel_readl(atchan, SADDR), channel_readl(atchan, DADDR), @@ -412,7 +407,7 @@ static void vdbg_dump_regs(struct at_dma_chan *atchan) {} static void atc_dump_lli(struct at_dma_chan *atchan, struct at_lli *lli) { - dev_crit(chan2dev(&atchan->vc.chan), + dev_crit(vchan_chan_dev(&atchan->vc), "desc: s%pad d%pad ctrl0x%x:0x%x l%pad\n", &lli->saddr, &lli->daddr, lli->ctrla, lli->ctrlb, &lli->dscr); @@ -790,8 +785,8 @@ static void atc_handle_error(struct at_dma_chan *atchan, unsigned int i) * controller flagged an error instead of scribbling over * random memory locations. */ - dev_crit(chan2dev(&atchan->vc.chan), "Bad descriptor submitted for DMA!\n"); - dev_crit(chan2dev(&atchan->vc.chan), "cookie: %d\n", + dev_crit(vchan_chan_dev(&atchan->vc), "Bad descriptor submitted for DMA!\n"); + dev_crit(vchan_chan_dev(&atchan->vc), "cookie: %d\n", desc->vd.tx.cookie); for (i = 0; i < desc->sglen; i++) atc_dump_lli(atchan, desc->sg[i].lli); @@ -886,7 +881,7 @@ atc_prep_dma_interleaved(struct dma_chan *chan, first = xt->sgl; - dev_info(chan2dev(chan), + dev_info(dmaengine_chan_dev(chan), "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n", __func__, &xt->src_start, &xt->dst_start, xt->numf, xt->frame_size, flags); @@ -903,7 +898,7 @@ atc_prep_dma_interleaved(struct dma_chan *chan, if ((chunk->size != xt->sgl->size) || (dmaengine_get_dst_icg(xt, chunk) != dmaengine_get_dst_icg(xt, first)) || (dmaengine_get_src_icg(xt, chunk) != dmaengine_get_src_icg(xt, first))) { - dev_err(chan2dev(chan), + dev_err(dmaengine_chan_dev(chan), "%s: the controller can transfer only identical chunks\n", __func__); return NULL; @@ -916,7 +911,7 @@ atc_prep_dma_interleaved(struct dma_chan *chan, xfer_count = len >> dwidth; if (xfer_count > ATC_BTSIZE_MAX) { - dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__); + dev_err(dmaengine_chan_dev(chan), "%s: buffer is too big\n", __func__); return NULL; } @@ -983,11 +978,11 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, u32 ctrla; u32 ctrlb; - dev_dbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n", + dev_dbg(dmaengine_chan_dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n", &dest, &src, len, flags); if (unlikely(!len)) { - dev_err(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); + dev_err(dmaengine_chan_dev(chan), "prep_dma_memcpy: length is zero!\n"); return NULL; } @@ -1062,7 +1057,7 @@ static int atdma_create_memset_lli(struct dma_chan *chan, xfer_count = len >> 2; if (xfer_count > ATC_BTSIZE_MAX) { - dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__); + dev_err(dmaengine_chan_dev(chan), "%s: buffer is too big\n", __func__); return -EINVAL; } @@ -1097,28 +1092,28 @@ atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); struct at_desc *desc; - void __iomem *vaddr; + u32 *vaddr; dma_addr_t paddr; char fill_pattern; int ret; - dev_vdbg(chan2dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__, + dev_vdbg(dmaengine_chan_dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__, &dest, value, len, flags); if (unlikely(!len)) { - dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: length is zero!\n", __func__); return NULL; } if (!is_dma_fill_aligned(chan->device, dest, 0, len)) { - dev_dbg(chan2dev(chan), "%s: buffer is not aligned\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: buffer is not aligned\n", __func__); return NULL; } vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr); if (!vaddr) { - dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", + dev_err(dmaengine_chan_dev(chan), "%s: couldn't allocate buffer\n", __func__); return NULL; } @@ -1126,10 +1121,10 @@ atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, /* Only the first byte of value is to be used according to dmaengine */ fill_pattern = (char)value; - *(u32*)vaddr = (fill_pattern << 24) | - (fill_pattern << 16) | - (fill_pattern << 8) | - fill_pattern; + *vaddr = (fill_pattern << 24) | + (fill_pattern << 16) | + (fill_pattern << 8) | + fill_pattern; desc = kzalloc_flex(*desc, sg, 1, GFP_ATOMIC); if (!desc) @@ -1168,28 +1163,28 @@ atc_prep_dma_memset_sg(struct dma_chan *chan, struct at_dma *atdma = to_at_dma(chan->device); struct at_desc *desc; struct scatterlist *sg; - void __iomem *vaddr; + u32 *vaddr; dma_addr_t paddr; size_t total_len = 0; int i; int ret; - dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%x f0x%lx\n", __func__, + dev_vdbg(dmaengine_chan_dev(chan), "%s: v0x%x l0x%x f0x%lx\n", __func__, value, sg_len, flags); if (unlikely(!sgl || !sg_len)) { - dev_dbg(chan2dev(chan), "%s: scatterlist is empty!\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: scatterlist is empty!\n", __func__); return NULL; } vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr); if (!vaddr) { - dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", + dev_err(dmaengine_chan_dev(chan), "%s: couldn't allocate buffer\n", __func__); return NULL; } - *(u32*)vaddr = value; + *vaddr = value; desc = kzalloc_flex(*desc, sg, sg_len, GFP_ATOMIC); if (!desc) @@ -1200,11 +1195,11 @@ atc_prep_dma_memset_sg(struct dma_chan *chan, dma_addr_t dest = sg_dma_address(sg); size_t len = sg_dma_len(sg); - dev_vdbg(chan2dev(chan), "%s: d%pad, l0x%zx\n", + dev_vdbg(dmaengine_chan_dev(chan), "%s: d%pad, l0x%zx\n", __func__, &dest, len); if (!is_dma_fill_aligned(chan->device, dest, 0, len)) { - dev_err(chan2dev(chan), "%s: buffer is not aligned\n", + dev_err(dmaengine_chan_dev(chan), "%s: buffer is not aligned\n", __func__); goto err_free_desc; } @@ -1264,13 +1259,13 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, struct scatterlist *sg; size_t total_len = 0; - dev_vdbg(chan2dev(chan), "prep_slave_sg (%d): %s f0x%lx\n", + dev_vdbg(dmaengine_chan_dev(chan), "prep_slave_sg (%d): %s f0x%lx\n", sg_len, direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE", flags); if (unlikely(!atslave || !sg_len)) { - dev_dbg(chan2dev(chan), "prep_slave_sg: sg length is zero!\n"); + dev_dbg(dmaengine_chan_dev(chan), "prep_slave_sg: sg length is zero!\n"); return NULL; } @@ -1310,7 +1305,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, mem = sg_dma_address(sg); len = sg_dma_len(sg); if (unlikely(!len)) { - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "prep_slave_sg: sg(%d) data length is zero\n", i); goto err; } @@ -1359,7 +1354,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, mem = sg_dma_address(sg); len = sg_dma_len(sg); if (unlikely(!len)) { - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "prep_slave_sg: sg(%d) data length is zero\n", i); goto err; } @@ -1392,7 +1387,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return vchan_tx_prep(&atchan->vc, &desc->vd, flags); err_desc_get: - dev_err(chan2dev(chan), "not enough descriptors available\n"); + dev_err(dmaengine_chan_dev(chan), "not enough descriptors available\n"); err: atdma_desc_free(&desc->vd); return NULL; @@ -1503,19 +1498,19 @@ atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, unsigned int periods = buf_len / period_len; unsigned int i; - dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%zu/%zu)\n", + dev_vdbg(dmaengine_chan_dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%zu/%zu)\n", direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE", &buf_addr, periods, buf_len, period_len); if (unlikely(!atslave || !buf_len || !period_len)) { - dev_dbg(chan2dev(chan), "prep_dma_cyclic: length is zero!\n"); + dev_dbg(dmaengine_chan_dev(chan), "prep_dma_cyclic: length is zero!\n"); return NULL; } was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status); if (was_cyclic) { - dev_dbg(chan2dev(chan), "prep_dma_cyclic: channel in use!\n"); + dev_dbg(dmaengine_chan_dev(chan), "prep_dma_cyclic: channel in use!\n"); return NULL; } @@ -1561,7 +1556,7 @@ static int atc_config(struct dma_chan *chan, { struct at_dma_chan *atchan = to_at_dma_chan(chan); - dev_vdbg(chan2dev(chan), "%s\n", __func__); + dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__); /* Check if it is chan is configured for slave transfers */ if (!chan->private) @@ -1582,7 +1577,7 @@ static int atc_pause(struct dma_chan *chan) int chan_id = atchan->vc.chan.chan_id; unsigned long flags; - dev_vdbg(chan2dev(chan), "%s\n", __func__); + dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__); spin_lock_irqsave(&atchan->vc.lock, flags); @@ -1601,7 +1596,7 @@ static int atc_resume(struct dma_chan *chan) int chan_id = atchan->vc.chan.chan_id; unsigned long flags; - dev_vdbg(chan2dev(chan), "%s\n", __func__); + dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__); if (!atc_chan_is_paused(atchan)) return 0; @@ -1625,7 +1620,7 @@ static int atc_terminate_all(struct dma_chan *chan) LIST_HEAD(list); - dev_vdbg(chan2dev(chan), "%s\n", __func__); + dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__); /* * This is only called when something went wrong elsewhere, so @@ -1691,13 +1686,13 @@ atc_tx_status(struct dma_chan *chan, spin_unlock_irqrestore(&atchan->vc.lock, flags); if (unlikely(ret < 0)) { - dev_vdbg(chan2dev(chan), "get residual bytes error\n"); + dev_vdbg(dmaengine_chan_dev(chan), "get residual bytes error\n"); return DMA_ERROR; } else { dma_set_residue(txstate, residue); } - dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %u\n", + dev_vdbg(dmaengine_chan_dev(chan), "tx_status %d: cookie = %d residue = %u\n", dma_status, cookie, residue); return dma_status; @@ -1729,11 +1724,11 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) struct at_dma_slave *atslave; u32 cfg; - dev_vdbg(chan2dev(chan), "alloc_chan_resources\n"); + dev_vdbg(dmaengine_chan_dev(chan), "alloc_chan_resources\n"); /* ASSERT: channel is idle */ if (atc_chan_is_enabled(atchan)) { - dev_dbg(chan2dev(chan), "DMA channel not idle ?\n"); + dev_dbg(dmaengine_chan_dev(chan), "DMA channel not idle ?\n"); return -EIO; } @@ -1782,7 +1777,7 @@ static void atc_free_chan_resources(struct dma_chan *chan) chan->private = NULL; } - dev_vdbg(chan2dev(chan), "free_chan_resources: done\n"); + dev_vdbg(dmaengine_chan_dev(chan), "free_chan_resources: done\n"); } #ifdef CONFIG_OF @@ -1940,6 +1935,20 @@ static void at_dma_off(struct at_dma *atdma) cpu_relax(); } +static void at_dma_cleanup_channels(struct at_dma *atdma) +{ + struct dma_chan *chan, *_chan; + int i = 0; + + list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, + device_node) { + /* Disable interrupts */ + atc_disable_chan_irq(atdma, i++); + tasklet_kill(&to_at_dma_chan(chan)->vc.task); + list_del(&chan->device_node); + } +} + static int __init at_dma_probe(struct platform_device *pdev) { struct at_dma *atdma; @@ -1980,20 +1989,16 @@ static int __init at_dma_probe(struct platform_device *pdev) atdma->dma_device.cap_mask = plat_dat->cap_mask; atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1; - atdma->clk = devm_clk_get(&pdev->dev, "dma_clk"); + atdma->clk = devm_clk_get_enabled(&pdev->dev, "dma_clk"); if (IS_ERR(atdma->clk)) return PTR_ERR(atdma->clk); - err = clk_prepare_enable(atdma->clk); - if (err) - return err; - /* force dma off, just in case */ at_dma_off(atdma); err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma); if (err) - goto err_irq; + return err; platform_set_drvdata(pdev, atdma); @@ -2105,38 +2110,30 @@ static int __init at_dma_probe(struct platform_device *pdev) err_of_dma_controller_register: dma_async_device_unregister(&atdma->dma_device); err_dma_async_device_register: + disable_irq(platform_get_irq(pdev, 0)); + at_dma_cleanup_channels(atdma); dma_pool_destroy(atdma->memset_pool); err_memset_pool_create: dma_pool_destroy(atdma->lli_pool); err_desc_pool_create: - free_irq(platform_get_irq(pdev, 0), atdma); -err_irq: - clk_disable_unprepare(atdma->clk); + free_irq(irq, atdma); return err; } static void at_dma_remove(struct platform_device *pdev) { struct at_dma *atdma = platform_get_drvdata(pdev); - struct dma_chan *chan, *_chan; at_dma_off(atdma); if (pdev->dev.of_node) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&atdma->dma_device); - dma_pool_destroy(atdma->memset_pool); - dma_pool_destroy(atdma->lli_pool); free_irq(platform_get_irq(pdev, 0), atdma); - list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, - device_node) { - /* Disable interrupts */ - atc_disable_chan_irq(atdma, chan->chan_id); - list_del(&chan->device_node); - } - - clk_disable_unprepare(atdma->clk); + at_dma_cleanup_channels(atdma); + dma_pool_destroy(atdma->memset_pool); + dma_pool_destroy(atdma->lli_pool); } static void at_dma_shutdown(struct platform_device *pdev) @@ -2169,7 +2166,7 @@ static void atc_suspend_cyclic(struct at_dma_chan *atchan) /* Channel should be paused by user * do it anyway even if it is not done already */ if (!atc_chan_is_paused(atchan)) { - dev_warn(chan2dev(chan), + dev_warn(dmaengine_chan_dev(chan), "cyclic channel not paused, should be done by channel user\n"); atc_pause(chan); } diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index 901971e8bae6..d40a8b40dddc 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -324,11 +324,6 @@ static inline struct at_xdmac_chan *to_at_xdmac_chan(struct dma_chan *dchan) return container_of(dchan, struct at_xdmac_chan, chan); } -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - static inline struct at_xdmac *to_at_xdmac(struct dma_device *ddev) { return container_of(ddev, struct at_xdmac, dma); @@ -459,7 +454,7 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, if (ret < 0) return; - dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first); + dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first); /* Set transfer as active to not try to start it again. */ first->active_xfer = true; @@ -496,7 +491,7 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, | AT_XDMAC_CNDC_NDE; at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg); - dev_vdbg(chan2dev(&atchan->chan), + dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n", __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC), at_xdmac_chan_read(atchan, AT_XDMAC_CNDA), @@ -524,12 +519,12 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, at_xdmac_chan_write(atchan, AT_XDMAC_CIE, reg | AT_XDMAC_CIE_LIE); at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask); - dev_vdbg(chan2dev(&atchan->chan), + dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: enable channel (0x%08x)\n", __func__, atchan->mask); wmb(); at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); - dev_vdbg(chan2dev(&atchan->chan), + dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n", __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC), at_xdmac_chan_read(atchan, AT_XDMAC_CNDA), @@ -552,7 +547,7 @@ static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx) list_add_tail(&desc->xfer_node, &atchan->xfers_list); spin_unlock_irqrestore(&atchan->lock, irqflags); - dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n", + dev_vdbg(dmaengine_chan_dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n", __func__, atchan, desc); return cookie; @@ -612,7 +607,7 @@ static void at_xdmac_queue_desc(struct dma_chan *chan, prev->lld.mbr_nda = desc->tx_dma_desc.phys; prev->lld.mbr_ubc |= AT_XDMAC_MBR_UBC_NDE; - dev_dbg(chan2dev(chan), "%s: chain lld: prev=0x%p, mbr_nda=%pad\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: chain lld: prev=0x%p, mbr_nda=%pad\n", __func__, prev, &prev->lld.mbr_nda); } @@ -624,7 +619,7 @@ static inline void at_xdmac_increment_block_count(struct dma_chan *chan, desc->lld.mbr_bc++; - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: incrementing the block count of the desc 0x%p\n", __func__, desc); } @@ -680,13 +675,13 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan, csize = ffs(atchan->sconfig.src_maxburst) - 1; if (csize < 0) { - dev_err(chan2dev(chan), "invalid src maxburst value\n"); + dev_err(dmaengine_chan_dev(chan), "invalid src maxburst value\n"); return -EINVAL; } atchan->cfg |= AT_XDMAC_CC_CSIZE(csize); dwidth = ffs(atchan->sconfig.src_addr_width) - 1; if (dwidth < 0) { - dev_err(chan2dev(chan), "invalid src addr width value\n"); + dev_err(dmaengine_chan_dev(chan), "invalid src addr width value\n"); return -EINVAL; } atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth); @@ -705,19 +700,19 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan, csize = ffs(atchan->sconfig.dst_maxburst) - 1; if (csize < 0) { - dev_err(chan2dev(chan), "invalid src maxburst value\n"); + dev_err(dmaengine_chan_dev(chan), "invalid src maxburst value\n"); return -EINVAL; } atchan->cfg |= AT_XDMAC_CC_CSIZE(csize); dwidth = ffs(atchan->sconfig.dst_addr_width) - 1; if (dwidth < 0) { - dev_err(chan2dev(chan), "invalid dst addr width value\n"); + dev_err(dmaengine_chan_dev(chan), "invalid dst addr width value\n"); return -EINVAL; } atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth); } - dev_dbg(chan2dev(chan), "%s: cfg=0x%08x\n", __func__, atchan->cfg); + dev_dbg(dmaengine_chan_dev(chan), "%s: cfg=0x%08x\n", __func__, atchan->cfg); return 0; } @@ -746,7 +741,7 @@ static int at_xdmac_set_slave_config(struct dma_chan *chan, struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); if (at_xdmac_check_slave_config(sconfig)) { - dev_err(chan2dev(chan), "invalid slave configuration\n"); + dev_err(dmaengine_chan_dev(chan), "invalid slave configuration\n"); return -EINVAL; } @@ -772,11 +767,11 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return NULL; if (!is_slave_direction(direction)) { - dev_err(chan2dev(chan), "invalid DMA direction\n"); + dev_err(dmaengine_chan_dev(chan), "invalid DMA direction\n"); return NULL; } - dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n", __func__, sg_len, direction == DMA_MEM_TO_DEV ? "to device" : "from device", flags); @@ -795,15 +790,15 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, len = sg_dma_len(sg); mem = sg_dma_address(sg); if (unlikely(!len)) { - dev_err(chan2dev(chan), "sg data length is zero\n"); + dev_err(dmaengine_chan_dev(chan), "sg data length is zero\n"); goto spin_unlock; } - dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n", __func__, i, len, mem); desc = at_xdmac_get_desc(atchan); if (!desc) { - dev_err(chan2dev(chan), "can't get descriptor\n"); + dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n"); if (first) list_splice_tail_init(&first->descs_list, &atchan->free_descs_list); @@ -828,7 +823,7 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, | (len >> fixed_dwidth); /* microblock length */ desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) | AT_XDMAC_CC_DWIDTH(fixed_dwidth); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n", __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc); @@ -840,7 +835,7 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!first) first = desc; - dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", __func__, desc, first); list_add_tail(&desc->desc_node, &first->descs_list); xfer_size += len; @@ -869,17 +864,17 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, int i; unsigned long irqflags; - dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n", __func__, &buf_addr, buf_len, period_len, direction == DMA_MEM_TO_DEV ? "mem2per" : "per2mem", flags); if (!is_slave_direction(direction)) { - dev_err(chan2dev(chan), "invalid DMA direction\n"); + dev_err(dmaengine_chan_dev(chan), "invalid DMA direction\n"); return NULL; } if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) { - dev_err(chan2dev(chan), "channel currently used\n"); + dev_err(dmaengine_chan_dev(chan), "channel currently used\n"); return NULL; } @@ -892,7 +887,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, spin_lock_irqsave(&atchan->lock, irqflags); desc = at_xdmac_get_desc(atchan); if (!desc) { - dev_err(chan2dev(chan), "can't get descriptor\n"); + dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n"); if (first) list_splice_tail_init(&first->descs_list, &atchan->free_descs_list); @@ -900,7 +895,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, return NULL; } spin_unlock_irqrestore(&atchan->lock, irqflags); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: desc=0x%p, tx_dma_desc.phys=%pad\n", __func__, desc, &desc->tx_dma_desc.phys); @@ -917,7 +912,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, | AT_XDMAC_MBR_UBC_NSEN | period_len >> at_xdmac_get_dwidth(desc->lld.mbr_cfg); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n", __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc); @@ -929,7 +924,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, if (!first) first = desc; - dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", __func__, desc, first); list_add_tail(&desc->desc_node, &first->descs_list); } @@ -956,16 +951,16 @@ static inline u32 at_xdmac_align_width(struct dma_chan *chan, dma_addr_t addr) */ if (!(addr & 7)) { width = AT_XDMAC_CC_DWIDTH_DWORD; - dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: dwidth: double word\n", __func__); } else if (!(addr & 3)) { width = AT_XDMAC_CC_DWIDTH_WORD; - dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: dwidth: word\n", __func__); } else if (!(addr & 1)) { width = AT_XDMAC_CC_DWIDTH_HALFWORD; - dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: dwidth: half word\n", __func__); } else { width = AT_XDMAC_CC_DWIDTH_BYTE; - dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: dwidth: byte\n", __func__); } return width; @@ -1005,7 +1000,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan, dwidth = at_xdmac_align_width(chan, src | dst | chunk->size); if (chunk->size >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) { - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: chunk too big (%zu, max size %lu)...\n", __func__, chunk->size, AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth); @@ -1013,7 +1008,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan, } if (prev) - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "Adding items at the end of desc 0x%p\n", prev); if (xt->src_inc) { @@ -1034,7 +1029,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan, desc = at_xdmac_get_desc(atchan); spin_unlock_irqrestore(&atchan->lock, flags); if (!desc) { - dev_err(chan2dev(chan), "can't get descriptor\n"); + dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n"); return NULL; } @@ -1053,7 +1048,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan, | ublen; desc->lld.mbr_cfg = chan_cc; - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc, desc->lld.mbr_cfg); @@ -1087,7 +1082,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan, if ((xt->numf > 1) && (xt->frame_size > 1)) return NULL; - dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n", __func__, &xt->src_start, &xt->dst_start, xt->numf, xt->frame_size, flags); @@ -1106,7 +1101,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan, for (i = 0; i < xt->numf - 1; i++) at_xdmac_increment_block_count(chan, first); - dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", __func__, first, first); list_add_tail(&first->desc_node, &first->descs_list); } else { @@ -1122,7 +1117,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan, src_skip = chunk->size + src_icg; dst_skip = chunk->size + dst_icg; - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: chunk size=%zu, src icg=%zu, dst icg=%zu\n", __func__, chunk->size, src_icg, dst_icg); @@ -1140,7 +1135,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan, if (!first) first = desc; - dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", __func__, desc, first); list_add_tail(&desc->desc_node, &first->descs_list); @@ -1193,7 +1188,7 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, | AT_XDMAC_CC_TYPE_MEM_TRAN; unsigned long irqflags; - dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n", __func__, &src, &dest, len, flags); if (unlikely(!len)) @@ -1205,13 +1200,13 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, while (remaining_size) { struct at_xdmac_desc *desc = NULL; - dev_dbg(chan2dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size); + dev_dbg(dmaengine_chan_dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size); spin_lock_irqsave(&atchan->lock, irqflags); desc = at_xdmac_get_desc(atchan); spin_unlock_irqrestore(&atchan->lock, irqflags); if (!desc) { - dev_err(chan2dev(chan), "can't get descriptor\n"); + dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n"); if (first) list_splice_tail_init(&first->descs_list, &atchan->free_descs_list); @@ -1227,7 +1222,7 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, else xfer_size = remaining_size; - dev_dbg(chan2dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size); + dev_dbg(dmaengine_chan_dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size); /* Check remaining length and change data width if needed. */ dwidth = at_xdmac_align_width(chan, @@ -1246,7 +1241,7 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, | ublen; desc->lld.mbr_cfg = chan_cc; - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc, desc->lld.mbr_cfg); @@ -1258,7 +1253,7 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, if (!first) first = desc; - dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", __func__, desc, first); list_add_tail(&desc->desc_node, &first->descs_list); } @@ -1306,7 +1301,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, dwidth = at_xdmac_align_width(chan, dst_addr); if (len >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) { - dev_err(chan2dev(chan), + dev_err(dmaengine_chan_dev(chan), "%s: Transfer too large, aborting...\n", __func__); return NULL; @@ -1316,7 +1311,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, desc = at_xdmac_get_desc(atchan); spin_unlock_irqrestore(&atchan->lock, flags); if (!desc) { - dev_err(chan2dev(chan), "can't get descriptor\n"); + dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n"); return NULL; } @@ -1338,7 +1333,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, | ublen; desc->lld.mbr_cfg = chan_cc; - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: lld: mbr_da=%pad, mbr_ds=0x%08x, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", __func__, &desc->lld.mbr_da, desc->lld.mbr_ds, desc->lld.mbr_ubc, desc->lld.mbr_cfg); @@ -1353,7 +1348,7 @@ at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac_desc *desc; - dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%zu, pattern=0x%x, flags=0x%lx\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: dest=%pad, len=%zu, pattern=0x%x, flags=0x%lx\n", __func__, &dest, len, value, flags); if (unlikely(!len)) @@ -1386,12 +1381,12 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!sgl) return NULL; - dev_dbg(chan2dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n", __func__, sg_len, value, flags); /* Prepare descriptors. */ for_each_sg(sgl, sg, sg_len, i) { - dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n", __func__, &sg_dma_address(sg), sg_dma_len(sg), value, flags); desc = at_xdmac_memset_create_desc(chan, atchan, @@ -1433,7 +1428,7 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl, if (ppdesc && pdesc) { if ((stride == pstride) && (sg_dma_len(ppsg) == sg_dma_len(psg))) { - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: desc 0x%p can be merged with desc 0x%p\n", __func__, pdesc, ppdesc); @@ -1481,7 +1476,7 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl, */ list_add_tail(&desc->desc_node, &first->descs_list); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", __func__, desc, first); } @@ -1496,7 +1491,7 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl, */ if ((i == (sg_len - 1)) && sg_dma_len(psg) == sg_dma_len(sg)) { - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: desc 0x%p can be merged with desc 0x%p\n", __func__, desc, pdesc); @@ -1667,7 +1662,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, dma_set_residue(txstate, residue); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s: desc=0x%p, tx_dma_desc.phys=%pad, tx_status=%d, cookie=%d, residue=%d\n", __func__, desc, &desc->tx_dma_desc.phys, ret, cookie, residue); @@ -1690,7 +1685,7 @@ static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); - dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); + dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); if (!desc->active_xfer) at_xdmac_start_xfer(atchan, desc); } @@ -1701,7 +1696,7 @@ static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) struct dma_async_tx_descriptor *txd; spin_lock_irq(&atchan->lock); - dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n", + dev_dbg(dmaengine_chan_dev(&atchan->chan), "%s: status=0x%08x\n", __func__, atchan->irq_status); if (list_empty(&atchan->xfers_list)) { spin_unlock_irq(&atchan->lock); @@ -1733,11 +1728,11 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) * descriptors queued (if any). */ if (atchan->irq_status & AT_XDMAC_CIS_RBEIS) - dev_err(chan2dev(&atchan->chan), "read bus error!!!"); + dev_err(dmaengine_chan_dev(&atchan->chan), "read bus error!!!"); if (atchan->irq_status & AT_XDMAC_CIS_WBEIS) - dev_err(chan2dev(&atchan->chan), "write bus error!!!"); + dev_err(dmaengine_chan_dev(&atchan->chan), "write bus error!!!"); if (atchan->irq_status & AT_XDMAC_CIS_ROIS) - dev_err(chan2dev(&atchan->chan), "request overflow error!!!"); + dev_err(dmaengine_chan_dev(&atchan->chan), "request overflow error!!!"); /* Channel must be disabled first as it's not done automatically */ at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); @@ -1749,7 +1744,7 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) xfer_node); /* Print bad descriptor's details if needed */ - dev_dbg(chan2dev(&atchan->chan), + dev_dbg(dmaengine_chan_dev(&atchan->chan), "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n", __func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da, bad_desc->lld.mbr_ubc); @@ -1775,7 +1770,7 @@ static void at_xdmac_tasklet(struct tasklet_struct *t) spin_lock_irq(&atchan->lock); - dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n", + dev_dbg(dmaengine_chan_dev(&atchan->chan), "%s: status=0x%08x\n", __func__, atchan->irq_status); if (!(atchan->irq_status & AT_XDMAC_CIS_LIS) && @@ -1789,9 +1784,9 @@ static void at_xdmac_tasklet(struct tasklet_struct *t) desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); - dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); + dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); if (!desc->active_xfer) { - dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting"); + dev_err(dmaengine_chan_dev(&atchan->chan), "Xfer not active: exiting"); spin_unlock_irq(&atchan->lock); return; } @@ -1852,7 +1847,7 @@ static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id) dev_vdbg(atxdmac->dma.dev, "%s: chan%d: imr=0x%x, status=0x%x\n", __func__, i, chan_imr, chan_status); - dev_vdbg(chan2dev(&atchan->chan), + dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n", __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC), @@ -1879,7 +1874,7 @@ static void at_xdmac_issue_pending(struct dma_chan *chan) struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); unsigned long flags; - dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__); + dev_dbg(dmaengine_chan_dev(&atchan->chan), "%s\n", __func__); spin_lock_irqsave(&atchan->lock, flags); at_xdmac_advance_work(atchan); @@ -1895,7 +1890,7 @@ static int at_xdmac_device_config(struct dma_chan *chan, int ret; unsigned long flags; - dev_dbg(chan2dev(chan), "%s\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s\n", __func__); spin_lock_irqsave(&atchan->lock, flags); ret = at_xdmac_set_slave_config(chan, config); @@ -1931,7 +1926,7 @@ static int at_xdmac_device_pause(struct dma_chan *chan) unsigned long flags; int ret; - dev_dbg(chan2dev(chan), "%s\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s\n", __func__); if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status)) return 0; @@ -1971,7 +1966,7 @@ static int at_xdmac_device_resume(struct dma_chan *chan) unsigned long flags; int ret; - dev_dbg(chan2dev(chan), "%s\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s\n", __func__); ret = pm_runtime_resume_and_get(atxdmac->dev); if (ret < 0) @@ -2004,7 +1999,7 @@ static int at_xdmac_device_terminate_all(struct dma_chan *chan) unsigned long flags; int ret; - dev_dbg(chan2dev(chan), "%s\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s\n", __func__); ret = pm_runtime_resume_and_get(atxdmac->dev); if (ret < 0) @@ -2045,13 +2040,13 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) int i; if (at_xdmac_chan_is_enabled(atchan)) { - dev_err(chan2dev(chan), + dev_err(dmaengine_chan_dev(chan), "can't allocate channel resources (channel enabled)\n"); return -EIO; } if (!list_empty(&atchan->free_descs_list)) { - dev_err(chan2dev(chan), + dev_err(dmaengine_chan_dev(chan), "can't allocate channel resources (channel not free from a previous use)\n"); return -EIO; } @@ -2060,11 +2055,11 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) desc = at_xdmac_alloc_desc(chan, GFP_KERNEL); if (!desc) { if (i == 0) { - dev_warn(chan2dev(chan), + dev_warn(dmaengine_chan_dev(chan), "can't allocate any descriptors\n"); return -EIO; } - dev_warn(chan2dev(chan), + dev_warn(dmaengine_chan_dev(chan), "only %d descriptors have been allocated\n", i); break; } @@ -2073,7 +2068,7 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) dma_cookie_init(chan); - dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i); + dev_dbg(dmaengine_chan_dev(chan), "%s: allocated %d descriptors\n", __func__, i); return i; } @@ -2085,7 +2080,7 @@ static void at_xdmac_free_chan_resources(struct dma_chan *chan) struct at_xdmac_desc *desc, *_desc; list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) { - dev_dbg(chan2dev(chan), "%s: freeing descriptor %p\n", __func__, desc); + dev_dbg(dmaengine_chan_dev(chan), "%s: freeing descriptor %p\n", __func__, desc); list_del(&desc->desc_node); dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys); } @@ -2148,7 +2143,7 @@ static int __maybe_unused atmel_xdmac_suspend(struct device *dev) atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC); if (at_xdmac_chan_is_cyclic(atchan)) { if (!at_xdmac_chan_is_paused(atchan)) { - dev_warn(chan2dev(chan), "%s: channel %d not paused\n", + dev_warn(dmaengine_chan_dev(chan), "%s: channel %d not paused\n", __func__, chan->chan_id); at_xdmac_device_pause_internal(atchan); at_xdmac_runtime_suspend_descriptors(atchan); diff --git a/drivers/dma/bestcomm/Kconfig b/drivers/dma/bestcomm/Kconfig index 5dd437295964..e23ba4982d7e 100644 --- a/drivers/dma/bestcomm/Kconfig +++ b/drivers/dma/bestcomm/Kconfig @@ -5,7 +5,7 @@ config PPC_BESTCOMM tristate "Bestcomm DMA engine support" - depends on PPC_MPC52xx + depends on PPC_MPC52xx || (PPC && COMPILE_TEST) default n select PPC_LIB_RHEAP help diff --git a/drivers/dma/bestcomm/ata.c b/drivers/dma/bestcomm/ata.c index 502a45d76adc..40126f3f189e 100644 --- a/drivers/dma/bestcomm/ata.c +++ b/drivers/dma/bestcomm/ata.c @@ -55,7 +55,6 @@ bcom_ata_init(int queue_len, int maxbufsize) { struct bcom_task *tsk; struct bcom_ata_var *var; - struct bcom_ata_inc *inc; /* Prefetch breaks ATA DMA. Turn it off for ATA DMA */ bcom_disable_prefetch(); @@ -69,7 +68,6 @@ bcom_ata_init(int queue_len, int maxbufsize) bcom_ata_reset_bd(tsk); var = (struct bcom_ata_var *) bcom_task_var(tsk->tasknum); - inc = (struct bcom_ata_inc *) bcom_task_inc(tsk->tasknum); if (bcom_load_image(tsk->tasknum, bcom_ata_task)) { bcom_task_free(tsk); diff --git a/drivers/dma/bestcomm/bestcomm.c b/drivers/dma/bestcomm/bestcomm.c index 432b43520ddc..241a91936fe8 100644 --- a/drivers/dma/bestcomm/bestcomm.c +++ b/drivers/dma/bestcomm/bestcomm.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -365,13 +364,19 @@ bcom_engine_cleanup(void) static int mpc52xx_bcom_probe(struct platform_device *op) { struct device_node *ofn_sram; - struct resource res_bcom; + struct resource *res_bcom; + void __iomem *regs; int rv; /* Inform user we're ok so far */ printk(KERN_INFO "DMA: MPC52xx BestComm driver\n"); + /* Get, reserve & map io */ + regs = devm_platform_get_and_ioremap_resource(op, 0, &res_bcom); + if (IS_ERR(regs)) + return PTR_ERR(regs); + /* Get the bestcomm node */ of_node_get(op->dev.of_node); @@ -402,35 +407,13 @@ static int mpc52xx_bcom_probe(struct platform_device *op) /* Save the node */ bcom_eng->ofnode = op->dev.of_node; - /* Get, reserve & map io */ - if (of_address_to_resource(op->dev.of_node, 0, &res_bcom)) { - printk(KERN_ERR DRIVER_NAME ": " - "Can't get resource\n"); - rv = -EINVAL; - goto error_sramclean; - } - - if (!request_mem_region(res_bcom.start, resource_size(&res_bcom), - DRIVER_NAME)) { - printk(KERN_ERR DRIVER_NAME ": " - "Can't request registers region\n"); - rv = -EBUSY; - goto error_sramclean; - } - - bcom_eng->regs_base = res_bcom.start; - bcom_eng->regs = ioremap(res_bcom.start, sizeof(struct mpc52xx_sdma)); - if (!bcom_eng->regs) { - printk(KERN_ERR DRIVER_NAME ": " - "Can't map registers\n"); - rv = -ENOMEM; - goto error_release; - } + bcom_eng->regs = regs; + bcom_eng->regs_base = res_bcom->start; /* Now, do the real init */ rv = bcom_engine_init(); if (rv) - goto error_unmap; + goto error_sramclean; /* Done ! */ printk(KERN_INFO "DMA: MPC52xx BestComm engine @%08lx ok !\n", @@ -439,12 +422,9 @@ static int mpc52xx_bcom_probe(struct platform_device *op) return 0; /* Error path */ -error_unmap: - iounmap(bcom_eng->regs); -error_release: - release_mem_region(res_bcom.start, sizeof(struct mpc52xx_sdma)); error_sramclean: kfree(bcom_eng); + bcom_eng = NULL; bcom_sram_cleanup(); error_ofput: of_node_put(op->dev.of_node); @@ -463,10 +443,6 @@ static void mpc52xx_bcom_remove(struct platform_device *op) /* Cleanup SRAM */ bcom_sram_cleanup(); - /* Release regs */ - iounmap(bcom_eng->regs); - release_mem_region(bcom_eng->regs_base, sizeof(struct mpc52xx_sdma)); - /* Release the node */ of_node_put(bcom_eng->ofnode); diff --git a/drivers/dma/bestcomm/gen_bd.c b/drivers/dma/bestcomm/gen_bd.c index 8a24a5cbc263..3b4fabb4d12b 100644 --- a/drivers/dma/bestcomm/gen_bd.c +++ b/drivers/dma/bestcomm/gen_bd.c @@ -81,7 +81,7 @@ struct bcom_gen_bd_priv { /* Task support code */ /* ======================================================================== */ -struct bcom_task * +static struct bcom_task * bcom_gen_bd_rx_init(int queue_len, phys_addr_t fifo, int initiator, int ipr, int maxbufsize) { @@ -108,7 +108,6 @@ bcom_gen_bd_rx_init(int queue_len, phys_addr_t fifo, return tsk; } -EXPORT_SYMBOL_GPL(bcom_gen_bd_rx_init); int bcom_gen_bd_rx_reset(struct bcom_task *tsk) @@ -166,7 +165,7 @@ bcom_gen_bd_rx_release(struct bcom_task *tsk) EXPORT_SYMBOL_GPL(bcom_gen_bd_rx_release); -extern struct bcom_task * +static struct bcom_task * bcom_gen_bd_tx_init(int queue_len, phys_addr_t fifo, int initiator, int ipr) { @@ -192,7 +191,6 @@ bcom_gen_bd_tx_init(int queue_len, phys_addr_t fifo, return tsk; } -EXPORT_SYMBOL_GPL(bcom_gen_bd_tx_init); int bcom_gen_bd_tx_reset(struct bcom_task *tsk) @@ -254,17 +252,23 @@ EXPORT_SYMBOL_GPL(bcom_gen_bd_tx_release); */ /** - * bcom_psc_parameters - Bestcomm initialization value table for PSC devices + * struct bcom_psc_params - Bestcomm initialization value table for PSC devices + * @rx_initiator: RX initiator ID + * @rx_ipr: RX interrupt priority register value + * @tx_initiator: TX initiator ID + * @tx_ipr: TX interrupt priority register value * * This structure is only used internally. It is a lookup table for PSC * specific parameters to bestcomm tasks. */ -static struct bcom_psc_params { +struct bcom_psc_params { int rx_initiator; int rx_ipr; int tx_initiator; int tx_ipr; -} bcom_psc_params[] = { +}; + +static const struct bcom_psc_params bcom_psc_params[] = { [0] = { .rx_initiator = BCOM_INITIATOR_PSC1_RX, .rx_ipr = BCOM_IPR_PSC1_RX, @@ -315,7 +319,7 @@ static struct bcom_psc_params { struct bcom_task * bcom_psc_gen_bd_rx_init(unsigned psc_num, int queue_len, phys_addr_t fifo, int maxbufsize) { - if (psc_num >= MPC52xx_PSC_MAXNUM) + if (psc_num >= ARRAY_SIZE(bcom_psc_params)) return NULL; return bcom_gen_bd_rx_init(queue_len, fifo, @@ -336,6 +340,9 @@ EXPORT_SYMBOL_GPL(bcom_psc_gen_bd_rx_init); struct bcom_task * bcom_psc_gen_bd_tx_init(unsigned psc_num, int queue_len, phys_addr_t fifo) { + if (psc_num >= ARRAY_SIZE(bcom_psc_params)) + return NULL; + struct psc; return bcom_gen_bd_tx_init(queue_len, fifo, bcom_psc_params[psc_num].tx_initiator, diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index 6070dfdb7114..ac2094e65ecf 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -687,12 +687,12 @@ static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); if (dcs & JZ_DMA_DCS_AR) { - dev_warn(&jzchan->vchan.chan.dev->device, + dev_warn(vchan_chan_dev(&jzchan->vchan), "address error (DCS=0x%x)\n", dcs); } if (dcs & JZ_DMA_DCS_HLT) { - dev_warn(&jzchan->vchan.chan.dev->device, + dev_warn(vchan_chan_dev(&jzchan->vchan), "channel halt (DCS=0x%x)\n", dcs); } @@ -721,7 +721,7 @@ static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, } } } else { - dev_err(&jzchan->vchan.chan.dev->device, + dev_err(vchan_chan_dev(&jzchan->vchan), "channel IRQ with no active transfer\n"); } @@ -760,12 +760,12 @@ static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan) { struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); - jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), + jzchan->desc_pool = dma_pool_create(dma_chan_name(chan), chan->device->dev, JZ_DMA_DESC_BLOCK_SIZE, PAGE_SIZE, 0); if (!jzchan->desc_pool) { - dev_err(&chan->dev->device, + dev_err(dmaengine_chan_dev(chan), "failed to allocate descriptor pool\n"); return -ENOMEM; } diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index c71763047126..34be53487a21 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -879,10 +879,10 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name) return chan; chan->slave = dev; - if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj, + if (sysfs_create_link(&dmaengine_chan_dev(chan)->kobj, &dev->kobj, DMA_SLAVE_NAME)) dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME); - if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name)) + if (sysfs_create_link(&dev->kobj, &dmaengine_chan_dev(chan)->kobj, chan->name)) dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name); return chan; @@ -928,7 +928,7 @@ void dma_release_channel(struct dma_chan *chan) dma_chan_put(chan); if (chan->slave) { - sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME); + sysfs_remove_link(&dmaengine_chan_dev(chan)->kobj, DMA_SLAVE_NAME); sysfs_remove_link(&chan->slave->kobj, chan->name); kfree(chan->name); chan->name = NULL; @@ -1089,22 +1089,25 @@ static int __dma_async_device_channel_register(struct dma_device *device, struct dma_chan *chan, const char *name) { + unsigned int id; int rc; chan->local = alloc_percpu(typeof(*chan->local)); if (!chan->local) return -ENOMEM; - chan->dev = kzalloc_obj(*chan->dev); - if (!chan->dev) { + chan->chan_dev = kzalloc_obj(*chan->chan_dev); + if (!chan->chan_dev) { rc = -ENOMEM; goto err_free_local; } - /* - * When the chan_id is a negative value, we are dynamically adding - * the channel. Otherwise we are static enumerating. - */ - chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL); + if (chan->chan_id & DMA_CHAN_ID_STATIC) { + id = chan->chan_id & ~DMA_CHAN_ID_STATIC; + chan->chan_id = ida_alloc_range(&device->chan_ida, id, id, + GFP_KERNEL); + } else { + chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL); + } if (chan->chan_id < 0) { pr_err("%s: unable to alloc ida for chan: %d\n", __func__, chan->chan_id); @@ -1112,17 +1115,17 @@ static int __dma_async_device_channel_register(struct dma_device *device, goto err_free_dev; } - chan->dev->device.class = &dma_devclass; - chan->dev->device.parent = device->dev; - chan->dev->chan = chan; - chan->dev->dev_id = device->dev_id; + dmaengine_chan_dev(chan)->class = &dma_devclass; + dmaengine_chan_dev(chan)->parent = device->dev; + chan->chan_dev->chan = chan; + chan->chan_dev->dev_id = device->dev_id; spin_lock_init(&chan->lock); if (!name) - dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id); + dev_set_name(dmaengine_chan_dev(chan), "dma%dchan%d", device->dev_id, chan->chan_id); else - dev_set_name(&chan->dev->device, "%s", name); - rc = device_register(&chan->dev->device); + dev_set_name(dmaengine_chan_dev(chan), "%s", name); + rc = device_register(dmaengine_chan_dev(chan)); if (rc) goto err_out_ida; chan->client_count = 0; @@ -1133,7 +1136,7 @@ static int __dma_async_device_channel_register(struct dma_device *device, err_out_ida: ida_free(&device->chan_ida, chan->chan_id); err_free_dev: - kfree(chan->dev); + kfree(chan->chan_dev); err_free_local: free_percpu(chan->local); chan->local = NULL; @@ -1166,10 +1169,10 @@ static void __dma_async_device_channel_unregister(struct dma_device *device, __func__, chan->client_count); mutex_lock(&dma_list_mutex); device->chancnt--; - chan->dev->chan = NULL; + chan->chan_dev->chan = NULL; mutex_unlock(&dma_list_mutex); ida_free(&device->chan_ida, chan->chan_id); - device_unregister(&chan->dev->device); + device_unregister(dmaengine_chan_dev(chan)); free_percpu(chan->local); } @@ -1301,9 +1304,9 @@ int dma_async_device_register(struct dma_device *device) if (chan->local == NULL) continue; mutex_lock(&dma_list_mutex); - chan->dev->chan = NULL; + chan->chan_dev->chan = NULL; mutex_unlock(&dma_list_mutex); - device_unregister(&chan->dev->device); + device_unregister(dmaengine_chan_dev(chan)); free_percpu(chan->local); } return rc; diff --git a/drivers/dma/dmaengine.h b/drivers/dma/dmaengine.h index 53f16d3f0029..42a3cfa7e597 100644 --- a/drivers/dma/dmaengine.h +++ b/drivers/dma/dmaengine.h @@ -182,6 +182,38 @@ dmaengine_desc_callback_valid(struct dmaengine_desc_callback *cb) struct dma_chan *dma_get_slave_channel(struct dma_chan *chan); struct dma_chan *dma_get_any_slave_channel(struct dma_device *device); +static inline char * +dmaengine_get_direction_text(enum dma_transfer_direction dir) +{ + switch (dir) { + case DMA_DEV_TO_MEM: + return "DEV_TO_MEM"; + case DMA_MEM_TO_DEV: + return "MEM_TO_DEV"; + case DMA_MEM_TO_MEM: + return "MEM_TO_MEM"; + case DMA_DEV_TO_DEV: + return "DEV_TO_DEV"; + default: + return "invalid"; + } +} + +static inline bool is_slave_direction(enum dma_transfer_direction direction) +{ + return (direction == DMA_MEM_TO_DEV) || (direction == DMA_DEV_TO_MEM) || + (direction == DMA_DEV_TO_DEV); +} + +int dma_async_device_register(struct dma_device *device); +int dmaenginem_async_device_register(struct dma_device *device); +void dma_async_device_unregister(struct dma_device *device); +int dma_async_device_channel_register(struct dma_device *device, + struct dma_chan *chan, + const char *name); +void dma_async_device_channel_unregister(struct dma_device *device, + struct dma_chan *chan); + #ifdef CONFIG_DEBUG_FS #include diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index eebed2474210..7520c78f55cc 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -7,6 +7,7 @@ * Author: Eugeniy Paltsev */ +#include #include #include #include @@ -101,23 +102,25 @@ static inline void axi_chan_config_write(struct axi_dma_chan *chan, { u32 cfg_lo, cfg_hi; - cfg_lo = (config->dst_multblk_type << CH_CFG_L_DST_MULTBLK_TYPE_POS | - config->src_multblk_type << CH_CFG_L_SRC_MULTBLK_TYPE_POS); if (chan->chip->dw->hdata->reg_map_8_channels && !chan->chip->dw->hdata->use_cfg2) { - cfg_hi = config->tt_fc << CH_CFG_H_TT_FC_POS | - config->hs_sel_src << CH_CFG_H_HS_SEL_SRC_POS | - config->hs_sel_dst << CH_CFG_H_HS_SEL_DST_POS | - config->src_per << CH_CFG_H_SRC_PER_POS | - config->dst_per << CH_CFG_H_DST_PER_POS | - config->prior << CH_CFG_H_PRIORITY_POS; + cfg_lo = FIELD_PREP(CH_CFG_L_DST_MULTBLK_TYPE, config->dst_multblk_type) | + FIELD_PREP(CH_CFG_L_SRC_MULTBLK_TYPE, config->src_multblk_type); + cfg_hi = FIELD_PREP(CH_CFG_H_TT_FC, config->tt_fc) | + FIELD_PREP(CH_CFG_H_HS_SEL_SRC, config->hs_sel_src) | + FIELD_PREP(CH_CFG_H_HS_SEL_DST, config->hs_sel_dst) | + FIELD_PREP(CH_CFG_H_SRC_PER, config->src_per) | + FIELD_PREP(CH_CFG_H_DST_PER, config->dst_per) | + FIELD_PREP(CH_CFG_H_PRIORITY, config->prior); } else { - cfg_lo |= config->src_per << CH_CFG2_L_SRC_PER_POS | - config->dst_per << CH_CFG2_L_DST_PER_POS; - cfg_hi = config->tt_fc << CH_CFG2_H_TT_FC_POS | - config->hs_sel_src << CH_CFG2_H_HS_SEL_SRC_POS | - config->hs_sel_dst << CH_CFG2_H_HS_SEL_DST_POS | - config->prior << CH_CFG2_H_PRIORITY_POS; + cfg_lo = FIELD_PREP(CH_CFG_L_DST_MULTBLK_TYPE, config->dst_multblk_type) | + FIELD_PREP(CH_CFG_L_SRC_MULTBLK_TYPE, config->src_multblk_type) | + FIELD_PREP(CH_CFG2_L_SRC_PER, config->src_per) | + FIELD_PREP(CH_CFG2_L_DST_PER, config->dst_per); + cfg_hi = FIELD_PREP(CH_CFG2_H_TT_FC, config->tt_fc) | + FIELD_PREP(CH_CFG2_H_HS_SEL_SRC, config->hs_sel_src) | + FIELD_PREP(CH_CFG2_H_HS_SEL_DST, config->hs_sel_dst) | + FIELD_PREP(CH_CFG2_H_PRIORITY, config->prior); } axi_chan_iowrite32(chan, CH_CFG_L, cfg_lo); axi_chan_iowrite32(chan, CH_CFG_H, cfg_hi); @@ -316,7 +319,7 @@ static struct axi_dma_lli *axi_desc_get(struct axi_dma_chan *chan, lli = dma_pool_zalloc(chan->desc_pool, GFP_NOWAIT, &phys); if (unlikely(!lli)) { - dev_err(chan2dev(chan), "%s: not enough descriptors available\n", + dev_err(vchan_chan_dev(&chan->vc), "%s: not enough descriptors available\n", axi_chan_name(chan)); return NULL; } @@ -342,7 +345,7 @@ static void axi_desc_put(struct axi_dma_desc *desc) kfree(desc->hw_desc); kfree(desc); atomic_sub(descs_put, &chan->descs_allocated); - dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n", + dev_vdbg(vchan_chan_dev(&chan->vc), "%s: %d descs put, %d still allocated\n", axi_chan_name(chan), descs_put, atomic_read(&chan->descs_allocated)); } @@ -431,7 +434,7 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, u8 lms = 0; /* Select AXI0 master for LLI fetching */ if (unlikely(axi_chan_is_hw_enable(chan))) { - dev_err(chan2dev(chan), "%s is non-idle!\n", + dev_err(vchan_chan_dev(&chan->vc), "%s is non-idle!\n", axi_chan_name(chan)); return; @@ -490,7 +493,7 @@ static void axi_chan_start_first_queued(struct axi_dma_chan *chan) return; desc = vd_to_axi_desc(vd); - dev_vdbg(chan2dev(chan), "%s: started %u\n", axi_chan_name(chan), + dev_vdbg(vchan_chan_dev(&chan->vc), "%s: started %u\n", axi_chan_name(chan), vd->tx.cookie); axi_chan_block_xfer_start(chan, desc); } @@ -524,23 +527,22 @@ static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) /* ASSERT: channel is idle */ if (axi_chan_is_hw_enable(chan)) { - dev_err(chan2dev(chan), "%s is non-idle!\n", + dev_err(vchan_chan_dev(&chan->vc), "%s is non-idle!\n", axi_chan_name(chan)); pm_runtime_put(chan->chip->dev); return -EBUSY; } /* LLI address must be aligned to a 64-byte boundary */ - chan->desc_pool = dma_pool_create(dev_name(chan2dev(chan)), - chan->chip->dev, + chan->desc_pool = dma_pool_create(dma_chan_name(dchan), chan->chip->dev, sizeof(struct axi_dma_lli), 64, 0); if (!chan->desc_pool) { - dev_err(chan2dev(chan), "No memory for descriptors\n"); + dev_err(vchan_chan_dev(&chan->vc), "No memory for descriptors\n"); pm_runtime_put(chan->chip->dev); return -ENOMEM; } - dev_vdbg(dchan2dev(dchan), "%s: allocating\n", axi_chan_name(chan)); + dev_vdbg(dmaengine_chan_dev(dchan), "%s: allocating\n", axi_chan_name(chan)); return 0; } @@ -551,7 +553,7 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan) /* ASSERT: channel is idle */ if (axi_chan_is_hw_enable(chan)) - dev_err(dchan2dev(dchan), "%s is non-idle!\n", + dev_err(dmaengine_chan_dev(dchan), "%s is non-idle!\n", axi_chan_name(chan)); axi_chan_disable(chan); @@ -561,7 +563,7 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan) dma_pool_destroy(chan->desc_pool); chan->desc_pool = NULL; - dev_vdbg(dchan2dev(dchan), + dev_vdbg(dmaengine_chan_dev(dchan), "%s: free resources, descriptor still allocated: %u\n", axi_chan_name(chan), atomic_read(&chan->descs_allocated)); @@ -677,19 +679,19 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan, case DMA_MEM_TO_DEV: reg_width = __ffs(chan->config.dst_addr_width); device_addr = chan->config.dst_addr; - ctllo = reg_width << CH_CTL_L_DST_WIDTH_POS | - mem_width << CH_CTL_L_SRC_WIDTH_POS | - DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS | - DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS; + ctllo = FIELD_PREP(CH_CTL_L_DST_WIDTH, reg_width) | + FIELD_PREP(CH_CTL_L_SRC_WIDTH, mem_width) | + FIELD_PREP(CH_CTL_L_DST_INC, DWAXIDMAC_CH_CTL_L_NOINC) | + FIELD_PREP(CH_CTL_L_SRC_INC, DWAXIDMAC_CH_CTL_L_INC); block_ts = len >> mem_width; break; case DMA_DEV_TO_MEM: reg_width = __ffs(chan->config.src_addr_width); device_addr = chan->config.src_addr; - ctllo = reg_width << CH_CTL_L_SRC_WIDTH_POS | - mem_width << CH_CTL_L_DST_WIDTH_POS | - DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS | - DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS; + ctllo = FIELD_PREP(CH_CTL_L_SRC_WIDTH, reg_width) | + FIELD_PREP(CH_CTL_L_DST_WIDTH, mem_width) | + FIELD_PREP(CH_CTL_L_DST_INC, DWAXIDMAC_CH_CTL_L_INC) | + FIELD_PREP(CH_CTL_L_SRC_INC, DWAXIDMAC_CH_CTL_L_NOINC); block_ts = len >> reg_width; break; default: @@ -706,10 +708,10 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan, ctlhi = CH_CTL_H_LLI_VALID; if (chan->chip->dw->hdata->restrict_axi_burst_len) { - burst_len = chan->chip->dw->hdata->axi_rw_burst_len; + burst_len = chan->chip->dw->hdata->axi_rw_burst_len - 1; ctlhi |= CH_CTL_H_ARLEN_EN | CH_CTL_H_AWLEN_EN | - burst_len << CH_CTL_H_ARLEN_POS | - burst_len << CH_CTL_H_AWLEN_POS; + FIELD_PREP(CH_CTL_H_ARLEN, burst_len) | + FIELD_PREP(CH_CTL_H_AWLEN, burst_len); } hw_desc->lli->ctl_hi = cpu_to_le32(ctlhi); @@ -724,8 +726,8 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan, hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1); - ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | - DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS; + ctllo |= FIELD_PREP(CH_CTL_L_DST_MSIZE, DWAXIDMAC_BURST_TRANS_LEN_4) | + FIELD_PREP(CH_CTL_L_SRC_MSIZE, DWAXIDMAC_BURST_TRANS_LEN_4); hw_desc->lli->ctl_lo = cpu_to_le32(ctllo); set_desc_src_master(hw_desc); @@ -930,7 +932,7 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, u64 llp = 0; u8 lms = 0; /* Select AXI0 master for LLI fetching */ - dev_dbg(chan2dev(chan), "%s: memcpy: src: %pad dst: %pad length: %zd flags: %#lx", + dev_dbg(vchan_chan_dev(&chan->vc), "%s: memcpy: src: %pad dst: %pad length: %zd flags: %#lx", axi_chan_name(chan), &src_adr, &dst_adr, len, flags); max_block_ts = chan->chip->dw->hdata->block_size[chan->id]; @@ -975,21 +977,21 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, reg = CH_CTL_H_LLI_VALID; if (chan->chip->dw->hdata->restrict_axi_burst_len) { - u32 burst_len = chan->chip->dw->hdata->axi_rw_burst_len; + u32 burst_len = chan->chip->dw->hdata->axi_rw_burst_len - 1; - reg |= (CH_CTL_H_ARLEN_EN | - burst_len << CH_CTL_H_ARLEN_POS | - CH_CTL_H_AWLEN_EN | - burst_len << CH_CTL_H_AWLEN_POS); + reg |= CH_CTL_H_ARLEN_EN | + FIELD_PREP(CH_CTL_H_ARLEN, burst_len) | + CH_CTL_H_AWLEN_EN | + FIELD_PREP(CH_CTL_H_AWLEN, burst_len); } hw_desc->lli->ctl_hi = cpu_to_le32(reg); - reg = (DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | - DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS | - xfer_width << CH_CTL_L_DST_WIDTH_POS | - xfer_width << CH_CTL_L_SRC_WIDTH_POS | - DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS | - DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS); + reg = FIELD_PREP(CH_CTL_L_DST_MSIZE, DWAXIDMAC_BURST_TRANS_LEN_4) | + FIELD_PREP(CH_CTL_L_SRC_MSIZE, DWAXIDMAC_BURST_TRANS_LEN_4) | + FIELD_PREP(CH_CTL_L_DST_WIDTH, xfer_width) | + FIELD_PREP(CH_CTL_L_SRC_WIDTH, xfer_width) | + FIELD_PREP(CH_CTL_L_DST_INC, DWAXIDMAC_CH_CTL_L_INC) | + FIELD_PREP(CH_CTL_L_SRC_INC, DWAXIDMAC_CH_CTL_L_INC); hw_desc->lli->ctl_lo = cpu_to_le32(reg); set_desc_src_master(hw_desc); @@ -1035,11 +1037,11 @@ static void axi_chan_dump_lli(struct axi_dma_chan *chan, struct axi_dma_hw_desc *desc) { if (!desc->lli) { - dev_err(dchan2dev(&chan->vc.chan), "NULL LLI\n"); + dev_err(vchan_chan_dev(&chan->vc), "NULL LLI\n"); return; } - dev_err(dchan2dev(&chan->vc.chan), + dev_err(vchan_chan_dev(&chan->vc), "SAR: 0x%llx DAR: 0x%llx LLP: 0x%llx BTS 0x%x CTL: 0x%x:%08x", le64_to_cpu(desc->lli->sar), le64_to_cpu(desc->lli->dar), @@ -1052,10 +1054,7 @@ static void axi_chan_dump_lli(struct axi_dma_chan *chan, static void axi_chan_list_dump_lli(struct axi_dma_chan *chan, struct axi_dma_desc *desc_head) { - int count = atomic_read(&chan->descs_allocated); - int i; - - for (i = 0; i < count; i++) + for (unsigned int i = 0; i < desc_head->nr_hw_descs; i++) axi_chan_dump_lli(chan, &desc_head->hw_desc[i]); } @@ -1071,7 +1070,7 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) /* The bad descriptor currently is in the head of vc list */ vd = vchan_next_desc(&chan->vc); if (!vd) { - dev_err(chan2dev(chan), "BUG: %s, IRQ with no descriptors\n", + dev_err(vchan_chan_dev(&chan->vc), "BUG: %s, IRQ with no descriptors\n", axi_chan_name(chan)); goto out; } @@ -1079,7 +1078,7 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) list_del(&vd->node); /* WARN about bad descriptor */ - dev_err(chan2dev(chan), + dev_err(vchan_chan_dev(&chan->vc), "Bad descriptor submitted for %s, cookie: %d, irq: 0x%08x\n", axi_chan_name(chan), vd->tx.cookie, status); axi_chan_list_dump_lli(chan, vd_to_axi_desc(vd)); @@ -1105,7 +1104,7 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) spin_lock_irqsave(&chan->vc.lock, flags); if (unlikely(axi_chan_is_hw_enable(chan))) { - dev_err(chan2dev(chan), "BUG: %s caught DWAXIDMAC_IRQ_DMA_TRF, but channel not idle!\n", + dev_err(vchan_chan_dev(&chan->vc), "BUG: %s caught DWAXIDMAC_IRQ_DMA_TRF, but channel not idle!\n", axi_chan_name(chan)); axi_chan_disable(chan); } @@ -1113,7 +1112,7 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) /* The completed descriptor currently is in the head of vc list */ vd = vchan_next_desc(&chan->vc); if (!vd) { - dev_err(chan2dev(chan), "BUG: %s, IRQ with no descriptors\n", + dev_err(vchan_chan_dev(&chan->vc), "BUG: %s, IRQ with no descriptors\n", axi_chan_name(chan)); goto out; } @@ -1193,7 +1192,7 @@ static int dma_chan_terminate_all(struct dma_chan *dchan) ret = readl_poll_timeout_atomic(chan->chip->regs + DMAC_CHEN, val, !(val & chan_active), 1000, 50000); if (ret == -ETIMEDOUT) - dev_warn(dchan2dev(dchan), + dev_warn(dmaengine_chan_dev(dchan), "%s failed to stop\n", axi_chan_name(chan)); if (chan->direction != DMA_MEM_TO_MEM) @@ -1210,7 +1209,7 @@ static int dma_chan_terminate_all(struct dma_chan *dchan) vchan_dma_desc_free_list(&chan->vc, &head); - dev_vdbg(dchan2dev(dchan), "terminated: %s\n", axi_chan_name(chan)); + dev_vdbg(dmaengine_chan_dev(dchan), "terminated: %s\n", axi_chan_name(chan)); return 0; } @@ -1320,8 +1319,7 @@ static int axi_dma_suspend(struct axi_dma_chip *chip) axi_dma_irq_disable(chip); axi_dma_disable(chip); - clk_disable_unprepare(chip->core_clk); - clk_disable_unprepare(chip->cfgr_clk); + clk_bulk_disable_unprepare(ARRAY_SIZE(chip->clks), chip->clks); return 0; } @@ -1330,11 +1328,7 @@ static int axi_dma_resume(struct axi_dma_chip *chip) { int ret; - ret = clk_prepare_enable(chip->cfgr_clk); - if (ret < 0) - return ret; - - ret = clk_prepare_enable(chip->core_clk); + ret = clk_bulk_prepare_enable(ARRAY_SIZE(chip->clks), chip->clks); if (ret < 0) return ret; @@ -1524,13 +1518,11 @@ static int dw_probe(struct platform_device *pdev) chip->dw->hdata->use_cfg2 = !!(flags & AXI_DMA_FLAG_USE_CFG2); - chip->core_clk = devm_clk_get(chip->dev, "core-clk"); - if (IS_ERR(chip->core_clk)) - return PTR_ERR(chip->core_clk); - - chip->cfgr_clk = devm_clk_get(chip->dev, "cfgr-clk"); - if (IS_ERR(chip->cfgr_clk)) - return PTR_ERR(chip->cfgr_clk); + chip->clks[0].id = "core-clk"; + chip->clks[1].id = "cfgr-clk"; + ret = devm_clk_bulk_get(chip->dev, ARRAY_SIZE(chip->clks), chip->clks); + if (ret) + return dev_err_probe(chip->dev, ret, "failed to get clocks\n"); ret = parse_device_properties(chip); if (ret) @@ -1640,10 +1632,13 @@ static void dw_remove(struct platform_device *pdev) struct dw_axi_dma *dw = chip->dw; struct axi_dma_chan *chan, *_chan; u32 i; + int ret; /* Enable clk before accessing to registers */ - clk_prepare_enable(chip->cfgr_clk); - clk_prepare_enable(chip->core_clk); + ret = clk_bulk_prepare_enable(ARRAY_SIZE(chip->clks), chip->clks); + if (ret) + dev_warn(chip->dev, "failed to enable clocks before remove: %d\n", + ret); axi_dma_irq_disable(chip); for (i = 0; i < dw->hdata->nr_channels; i++) { axi_chan_disable(&chip->dw->chan[i]); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index 67cc199e24d1..d8dd07efe60f 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -69,8 +69,7 @@ struct axi_dma_chip { int irq[DMAC_MAX_CHANNELS]; void __iomem *regs; void __iomem *apb_regs; - struct clk *core_clk; - struct clk *cfgr_clk; + struct clk_bulk_data clks[2]; struct dw_axi_dma *dw; }; @@ -119,16 +118,6 @@ struct axi_dma_chan_config { u8 hs_sel_src; }; -static inline struct device *dchan2dev(struct dma_chan *dchan) -{ - return &dchan->dev->device; -} - -static inline struct device *chan2dev(struct axi_dma_chan *chan) -{ - return &chan->vc.chan.dev->device; -} - static inline struct axi_dma_desc *vd_to_axi_desc(struct virt_dma_desc *vd) { return container_of(vd, struct axi_dma_desc, vd); @@ -234,9 +223,9 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) /* CH_CTL_H */ #define CH_CTL_H_ARLEN_EN BIT(6) -#define CH_CTL_H_ARLEN_POS 7 +#define CH_CTL_H_ARLEN GENMASK(14, 7) #define CH_CTL_H_AWLEN_EN BIT(15) -#define CH_CTL_H_AWLEN_POS 16 +#define CH_CTL_H_AWLEN GENMASK(23, 16) enum { DWAXIDMAC_ARWLEN_1 = 0, @@ -258,8 +247,8 @@ enum { /* CH_CTL_L */ #define CH_CTL_L_LAST_WRITE_EN BIT(30) -#define CH_CTL_L_DST_MSIZE_POS 18 -#define CH_CTL_L_SRC_MSIZE_POS 14 +#define CH_CTL_L_DST_MSIZE GENMASK(21, 18) +#define CH_CTL_L_SRC_MSIZE GENMASK(17, 14) enum { DWAXIDMAC_BURST_TRANS_LEN_1 = 0, @@ -274,11 +263,11 @@ enum { DWAXIDMAC_BURST_TRANS_LEN_1024 }; -#define CH_CTL_L_DST_WIDTH_POS 11 -#define CH_CTL_L_SRC_WIDTH_POS 8 +#define CH_CTL_L_DST_WIDTH GENMASK(13, 11) +#define CH_CTL_L_SRC_WIDTH GENMASK(10, 8) -#define CH_CTL_L_DST_INC_POS 6 -#define CH_CTL_L_SRC_INC_POS 4 +#define CH_CTL_L_DST_INC BIT(6) +#define CH_CTL_L_SRC_INC BIT(4) enum { DWAXIDMAC_CH_CTL_L_INC = 0, DWAXIDMAC_CH_CTL_L_NOINC @@ -288,17 +277,17 @@ enum { #define CH_CTL_L_SRC_MAST BIT(0) /* CH_CFG_H */ -#define CH_CFG_H_PRIORITY_POS 17 -#define CH_CFG_H_DST_PER_POS 12 -#define CH_CFG_H_SRC_PER_POS 7 -#define CH_CFG_H_HS_SEL_DST_POS 4 -#define CH_CFG_H_HS_SEL_SRC_POS 3 +#define CH_CFG_H_PRIORITY GENMASK(19, 17) +#define CH_CFG_H_DST_PER GENMASK(15, 12) +#define CH_CFG_H_SRC_PER GENMASK(10, 7) +#define CH_CFG_H_HS_SEL_DST BIT(4) +#define CH_CFG_H_HS_SEL_SRC BIT(3) enum { DWAXIDMAC_HS_SEL_HW = 0, DWAXIDMAC_HS_SEL_SW }; -#define CH_CFG_H_TT_FC_POS 0 +#define CH_CFG_H_TT_FC GENMASK(2, 0) enum { DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC = 0, DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC, @@ -311,8 +300,8 @@ enum { }; /* CH_CFG_L */ -#define CH_CFG_L_DST_MULTBLK_TYPE_POS 2 -#define CH_CFG_L_SRC_MULTBLK_TYPE_POS 0 +#define CH_CFG_L_DST_MULTBLK_TYPE GENMASK(3, 2) +#define CH_CFG_L_SRC_MULTBLK_TYPE GENMASK(1, 0) enum { DWAXIDMAC_MBLK_TYPE_CONTIGUOUS = 0, DWAXIDMAC_MBLK_TYPE_RELOAD, @@ -321,13 +310,13 @@ enum { }; /* CH_CFG2 */ -#define CH_CFG2_L_SRC_PER_POS 4 -#define CH_CFG2_L_DST_PER_POS 11 +#define CH_CFG2_L_SRC_PER GENMASK(9, 4) +#define CH_CFG2_L_DST_PER GENMASK(16, 11) -#define CH_CFG2_H_TT_FC_POS 0 -#define CH_CFG2_H_HS_SEL_SRC_POS 3 -#define CH_CFG2_H_HS_SEL_DST_POS 4 -#define CH_CFG2_H_PRIORITY_POS 20 +#define CH_CFG2_H_TT_FC GENMASK(2, 0) +#define CH_CFG2_H_HS_SEL_SRC BIT(3) +#define CH_CFG2_H_HS_SEL_DST BIT(4) +#define CH_CFG2_H_PRIORITY GENMASK(19, 15) /** * DW AXI DMA channel interrupts diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 03b2c2188351..b08bb31657c9 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -7,12 +7,14 @@ */ #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -30,9 +32,9 @@ struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) return container_of(vd, struct dw_edma_desc, vd); } -enum dw_edma_irq_event { - DW_EDMA_IRQ_DONE = BIT(0), - DW_EDMA_IRQ_ABORT = BIT(1), +enum dw_edma_deferred_event { + DW_EDMA_DEFERRED_DONE = BIT(0), + DW_EDMA_DEFERRED_ABORT = BIT(1), }; static inline @@ -51,13 +53,19 @@ dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst) { struct dw_edma_desc *desc; + /* + * For now, a descriptor that does not fit would stall the channel + * forever: reject it up front. + */ + if (!chan->non_ll && nburst > chan->ll_max - 1) + return NULL; + desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT); if (unlikely(!desc)) return NULL; desc->chan = chan; desc->nburst = nburst; - desc->cb = true; return desc; } @@ -67,38 +75,122 @@ static void vchan_free_desc(struct virt_dma_desc *vdesc) kfree(vd2dw_edma_desc(vdesc)); } -static void dw_edma_core_start(struct dw_edma_desc *desc, bool first) +/* Must be called with vc.lock held. */ +static void +dw_edma_set_request(struct dw_edma_chan *chan, enum dw_edma_request request) +{ + chan->request = request; +} + +static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, + enum dmaengine_tx_result result) +{ + u32 residue = 0; + struct dw_edma_desc *desc; + struct dmaengine_result *res; + + if (!vd->tx.callback_result) + return; + + desc = vd2dw_edma_desc(vd); + if (desc) { + residue = desc->alloc_sz; + + if (result == DMA_TRANS_NOERROR) + residue -= desc->burst[desc->start_burst - 1].xfer_sz; + else if (desc->done_burst) + residue -= desc->burst[desc->done_burst - 1].xfer_sz; + } + + res = &vd->tx_result; + res->result = result; + res->residue = residue; +} + +static void dw_edma_core_reset_ll(struct dw_edma_chan *chan) +{ + u32 i; + + chan->ll_head = 0; + chan->ll_done = 0; + /* Drop stale CB bits before reusing the circular LL ring. */ + for (i = 0; i < chan->ll_max; i++) + dw_edma_core_ll_clear(chan, i); + chan->cb = true; + + dw_edma_core_ll_link(chan, chan->ll_max, chan->cb, + chan->ll_region.paddr); + + dw_edma_core_ch_enable(chan); + chan->ll_valid = true; +} + +static u32 dw_edma_core_get_ll_dist(struct dw_edma_chan *chan, u32 from, u32 to) +{ + return (to + chan->ll_max - from) % chan->ll_max; +} + +static u32 dw_edma_core_get_used_num(struct dw_edma_chan *chan) +{ + return dw_edma_core_get_ll_dist(chan, chan->ll_done, chan->ll_head); +} + +static u32 dw_edma_core_get_free_num(struct dw_edma_chan *chan) +{ + /* Keep one data entry free so equal indices mean an empty ring. */ + return chan->ll_max - 1 - dw_edma_core_get_used_num(chan); +} + +static bool dw_edma_ll_pending(struct dw_edma_chan *chan) +{ + return chan->ll_head != chan->ll_done; +} + +static void dw_edma_core_ll_start(struct dw_edma_desc *desc) +{ + struct dw_edma_chan *chan = desc->chan; + size_t i; + u32 free; + + free = dw_edma_core_get_free_num(chan); + for (i = desc->start_burst; i < desc->nburst && free; i++, free--) { + /* + * Refresh the link element before filling the last data slot so + * the next lap has the updated CB value. + */ + if (chan->ll_head == chan->ll_max - 1) + dw_edma_core_ll_link(chan, chan->ll_max, chan->cb, + chan->ll_region.paddr); + + dw_edma_core_ll_data(chan, &desc->burst[i], + chan->ll_head, chan->cb, + i == desc->nburst - 1 || free == 1); + + chan->ll_head++; + + if (chan->ll_head == chan->ll_max) { + chan->cb = !chan->cb; + chan->ll_head = 0; + } + } + + desc->done_burst = desc->start_burst; + desc->start_burst = i; +} + +static void dw_edma_core_start(struct dw_edma_desc *desc) { struct dw_edma_chan *chan = desc->chan; - size_t i = 0; if (chan->non_ll) { - chan->dw->core->non_ll_start(chan, &desc->burst[desc->start_burst]); + chan->dw->core->non_ll_start(chan, + &desc->burst[desc->start_burst]); desc->done_burst = desc->start_burst; desc->start_burst += 1; return; } - for (i = 0; i + desc->start_burst < desc->nburst; i++) { - u32 idx = i + desc->start_burst; - - if (i == chan->ll_max) - break; - - dw_edma_core_ll_data(chan, &desc->burst[idx], - i, desc->cb, - idx == desc->nburst - 1 || i == chan->ll_max - 1); - } - - desc->done_burst = desc->start_burst; - desc->start_burst += i; - - dw_edma_core_ll_link(chan, i, desc->cb, chan->ll_region.paddr); - - if (first) - dw_edma_core_ch_enable(chan); - - dw_edma_core_ch_doorbell(chan); + dw_edma_core_ll_start(desc); } static int dw_edma_start_transfer(struct dw_edma_chan *chan) @@ -114,9 +206,10 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) if (!desc) return 0; - dw_edma_core_start(desc, !desc->start_burst); + if (!chan->non_ll && !chan->ll_valid) + dw_edma_core_reset_ll(chan); - desc->cb = !desc->cb; + dw_edma_core_start(desc); return 1; } @@ -150,6 +243,46 @@ static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan) dw_edma_terminate_vdesc_list(&chan->vc.desc_submitted); } +/* Must be called with vc.lock held after the channel has stopped. */ +static void dw_edma_finish_termination(struct dw_edma_chan *chan) +{ + dw_edma_terminate_all_descs(chan); + + /* Preserve a clean ring; resync only if entries remain published. */ + if (!chan->non_ll && dw_edma_ll_pending(chan)) + dw_edma_core_reset_ll(chan); + + dw_edma_set_request(chan, EDMA_REQ_NONE); + chan->status = EDMA_ST_IDLE; +} + +static void dw_edma_core_ll_sync(struct dw_edma_chan *chan) +{ + /* + * Remote controller registers and LL memory may be reached through + * different paths. Complete posted LL writes before the doorbell. + */ + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + readl(chan->ll_region.vaddr.io); +} + +/* Must be called with vc.lock held for an LL channel. */ +static void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan) +{ + dw_edma_core_ll_sync(chan); + dw_edma_core_do_ch_doorbell(chan); +} + +/* Must be called with vc.lock held. */ +static void dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan) +{ + if (chan->non_ll || chan->request != EDMA_REQ_NONE || + chan->status != EDMA_ST_BUSY || !dw_edma_ll_pending(chan)) + return; + + dw_edma_core_ch_doorbell(chan); +} + static void dw_edma_device_caps(struct dma_chan *dchan, struct dma_slave_caps *caps) { @@ -177,48 +310,76 @@ dw_edma_get_default_irq_mode(struct dw_edma_chan *chan) DW_EDMA_CH_IRQ_REMOTE; } +static int dw_edma_device_config_irq_mode(struct dw_edma_chan *chan, + enum dw_edma_ch_irq_mode mode) +{ + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) || + (mode != DW_EDMA_CH_IRQ_LOCAL && mode != DW_EDMA_CH_IRQ_REMOTE)) + return -EINVAL; + + guard(spinlock_irqsave)(&chan->vc.lock); + + if (chan->status != EDMA_ST_IDLE || chan->request != EDMA_REQ_NONE) + return -EBUSY; + + /* IRQ routing cannot change after the initial configuration. */ + if (chan->irq_mode == mode) + return 0; + + if (chan->configured) + return -EBUSY; + + chan->irq_mode = mode; + + return 0; +} + static int dw_edma_device_config(struct dma_chan *dchan, struct dma_slave_config *config) { + const struct dw_edma_chan_config *dw_config = config->peripheral_config; struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); - bool cfg_non_ll; - int non_ll = 0; + bool non_ll = false; + u32 flags = 0; + int ret; - chan->non_ll = false; - if (chan->dw->chip->mf == EDMA_MF_HDMA_NATIVE) { - if (config->peripheral_config && - config->peripheral_size != sizeof(int)) { - dev_err(dchan->device->dev, - "config param peripheral size mismatch\n"); + if (dw_config) { + if (config->peripheral_size != sizeof(*dw_config) || + dw_config->flags & ~(DW_EDMA_CH_CONFIG_NON_LL | + DW_EDMA_CH_CONFIG_IRQ_MODE)) + return -EINVAL; + flags = dw_config->flags; + } + + /* + * When there is no valid LLP base address available then the + * default DMA ops will use the non-LL mode. + * + * When LL mode is the default, clients can request non-LL mode + * through DW_EDMA_CH_CONFIG_NON_LL. + */ + non_ll = chan->dw->chip->mf == EDMA_MF_HDMA_NATIVE && + chan->dw->chip->cfg_non_ll; + + if (flags & DW_EDMA_CH_CONFIG_NON_LL) { + if (chan->dw->chip->mf != EDMA_MF_HDMA_NATIVE) + return -EINVAL; + + if (chan->dw->chip->cfg_non_ll && !dw_config->non_ll) { + dev_err(dchan->device->dev, "invalid configuration\n"); return -EINVAL; } - /* - * When there is no valid LLP base address available then the - * default DMA ops will use the non-LL mode. - * - * Cases where LL mode is enabled and client wants to use the - * non-LL mode then also client can do so via providing the - * peripheral_config param. - */ - cfg_non_ll = chan->dw->chip->cfg_non_ll; - if (config->peripheral_config) { - non_ll = *(int *)config->peripheral_config; - - if (cfg_non_ll && !non_ll) { - dev_err(dchan->device->dev, "invalid configuration\n"); - return -EINVAL; - } - } - - if (cfg_non_ll || non_ll) - chan->non_ll = true; - } else if (config->peripheral_config) { - dev_err(dchan->device->dev, - "peripheral config param applicable only for HDMA\n"); - return -EINVAL; + non_ll = dw_config->non_ll; } + if (flags & DW_EDMA_CH_CONFIG_IRQ_MODE) { + ret = dw_edma_device_config_irq_mode(chan, dw_config->irq_mode); + if (ret) + return ret; + } + + chan->non_ll = non_ll; memcpy(&chan->config, config, sizeof(*config)); chan->configured = true; @@ -253,7 +414,7 @@ static int dw_edma_device_pause(struct dma_chan *dchan) else if (chan->request != EDMA_REQ_NONE) err = -EPERM; else - chan->request = EDMA_REQ_PAUSE; + dw_edma_set_request(chan, EDMA_REQ_PAUSE); return err; } @@ -275,6 +436,7 @@ static int dw_edma_device_resume(struct dma_chan *dchan) chan->status = EDMA_ST_BUSY; if (!dw_edma_start_transfer(chan)) chan->status = EDMA_ST_IDLE; + dw_edma_core_ch_maybe_doorbell(chan); } return err; @@ -290,24 +452,22 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan) if (!chan->configured) { dw_edma_terminate_all_descs(chan); } else if (chan->status == EDMA_ST_PAUSE) { - dw_edma_terminate_all_descs(chan); - chan->status = EDMA_ST_IDLE; + dw_edma_finish_termination(chan); } else if (chan->status == EDMA_ST_IDLE) { - dw_edma_terminate_all_descs(chan); + dw_edma_finish_termination(chan); } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { /* * The channel is in a false BUSY state, probably didn't * receive or lost an interrupt */ - dw_edma_terminate_all_descs(chan); - chan->status = EDMA_ST_IDLE; + dw_edma_finish_termination(chan); } else if (chan->request > EDMA_REQ_PAUSE) { err = -EPERM; } else { - chan->request = EDMA_REQ_STOP; + dw_edma_set_request(chan, EDMA_REQ_STOP); } if (chan->status == EDMA_ST_IDLE) - chan->request = EDMA_REQ_NONE; + dw_edma_set_request(chan, EDMA_REQ_NONE); return err; } @@ -323,6 +483,7 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan) chan->status == EDMA_ST_IDLE) { chan->status = EDMA_ST_BUSY; dw_edma_start_transfer(chan); + dw_edma_core_ch_maybe_doorbell(chan); } spin_unlock_irqrestore(&chan->vc.lock, flags); } @@ -588,77 +749,62 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL)); } -static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, - enum dmaengine_tx_result result) +/* Must be called with vc.lock held. */ +static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan) { - u32 residue = 0; struct dw_edma_desc *desc; - struct dmaengine_result *res; + struct virt_dma_desc *vd; - if (!vd->tx.callback_result) + lockdep_assert_held(&chan->vc.lock); + + if (chan->status == EDMA_ST_PAUSE) return; - desc = vd2dw_edma_desc(vd); - if (desc) { - residue = desc->alloc_sz; + switch (chan->request) { + case EDMA_REQ_NONE: + case EDMA_REQ_PAUSE: + vd = vchan_next_desc(&chan->vc); + if (!vd) + break; - if (result == DMA_TRANS_NOERROR) - residue -= desc->burst[desc->start_burst - 1].xfer_sz; - else if (desc->done_burst) - residue -= desc->burst[desc->done_burst - 1].xfer_sz; + desc = vd2dw_edma_desc(vd); + if (desc->start_burst >= desc->nburst) { + dw_hdma_set_callback_result(vd, DMA_TRANS_NOERROR); + list_del(&vd->node); + vchan_cookie_complete(vd); + if (!chan->non_ll) + chan->ll_done = chan->ll_head; + } + + if (chan->request == EDMA_REQ_PAUSE) { + dw_edma_set_request(chan, EDMA_REQ_NONE); + chan->status = EDMA_ST_PAUSE; + break; + } + + chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; + break; + + case EDMA_REQ_STOP: + vd = vchan_next_desc(&chan->vc); + if (!vd) + break; + + dw_edma_finish_termination(chan); + break; + + default: + break; } - - res = &vd->tx_result; - res->result = result; - res->residue = residue; + dw_edma_core_ch_maybe_doorbell(chan); } static void dw_edma_done_interrupt(struct dw_edma_chan *chan) { - struct dw_edma_desc *desc; - struct virt_dma_desc *vd; unsigned long flags; spin_lock_irqsave(&chan->vc.lock, flags); - if (chan->status == EDMA_ST_PAUSE) { - spin_unlock_irqrestore(&chan->vc.lock, flags); - return; - } - - vd = vchan_next_desc(&chan->vc); - if (vd) { - switch (chan->request) { - case EDMA_REQ_NONE: - case EDMA_REQ_PAUSE: - desc = vd2dw_edma_desc(vd); - if (desc->start_burst >= desc->nburst) { - dw_hdma_set_callback_result(vd, - DMA_TRANS_NOERROR); - list_del(&vd->node); - vchan_cookie_complete(vd); - } - - if (chan->request == EDMA_REQ_PAUSE) { - chan->request = EDMA_REQ_NONE; - chan->status = EDMA_ST_PAUSE; - break; - } - - /* Continue transferring if there are remaining chunks or issued requests. - */ - chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; - break; - - case EDMA_REQ_STOP: - dw_edma_terminate_all_descs(chan); - chan->request = EDMA_REQ_NONE; - chan->status = EDMA_ST_IDLE; - break; - - default: - break; - } - } + dw_edma_done_interrupt_locked(chan); spin_unlock_irqrestore(&chan->vc.lock, flags); } @@ -676,7 +822,9 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) list_del(&vd->node); vchan_cookie_complete(vd); } - chan->request = EDMA_REQ_NONE; + if (!chan->non_ll) + dw_edma_core_reset_ll(chan); + dw_edma_set_request(chan, EDMA_REQ_NONE); chan->status = EDMA_ST_IDLE; spin_unlock_irqrestore(&chan->vc.lock, flags); } @@ -690,28 +838,32 @@ static void dw_edma_irq_work(struct work_struct *work) do { events = atomic_xchg(&chan->irq_pending, 0); - if (events & DW_EDMA_IRQ_DONE) + if (events & DW_EDMA_DEFERRED_DONE) dw_edma_done_interrupt(chan); - if (events & DW_EDMA_IRQ_ABORT) + if (events & DW_EDMA_DEFERRED_ABORT) dw_edma_abort_interrupt(chan); } while (atomic_read(&chan->irq_pending)); } static void dw_edma_queue_irq_work(struct dw_edma_chan *chan, - enum dw_edma_irq_event event) + unsigned int events) { - atomic_or(event, &chan->irq_pending); + atomic_or(events, &chan->irq_pending); queue_work(chan->dw->wq, &chan->irq_work); } -static void dw_edma_done_interrupt_deferred(struct dw_edma_chan *chan) +static void dw_edma_record_irq(struct dw_edma_chan *chan, unsigned int events) { - dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_DONE); -} + unsigned int pending = 0; -static void dw_edma_abort_interrupt_deferred(struct dw_edma_chan *chan) -{ - dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_ABORT); + if (events & (DW_EDMA_IRQ_DONE | DW_EDMA_IRQ_PROGRESS | + DW_EDMA_IRQ_STOP)) + pending |= DW_EDMA_DEFERRED_DONE; + if (events & DW_EDMA_IRQ_ABORT) + pending |= DW_EDMA_DEFERRED_ABORT; + + if (pending) + dw_edma_queue_irq_work(chan, pending); } static void dw_edma_emul_irq_ack(struct irq_data *d) @@ -812,8 +964,7 @@ static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data) struct dw_edma_irq *dw_irq = data; return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE, - dw_edma_done_interrupt_deferred, - dw_edma_abort_interrupt_deferred); + dw_edma_record_irq); } static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data) @@ -821,8 +972,7 @@ static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data) struct dw_edma_irq *dw_irq = data; return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ, - dw_edma_done_interrupt_deferred, - dw_edma_abort_interrupt_deferred); + dw_edma_record_irq); } static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) @@ -863,6 +1013,9 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) if (chan->status != EDMA_ST_IDLE) return -EBUSY; + /* The hardware context may have been invalidated while unowned. */ + chan->ll_valid = false; + return 0; } @@ -890,11 +1043,53 @@ static void dw_edma_wait_termination(struct dma_chan *dchan) "timeout waiting for channel termination\n"); } +static void dw_edma_synchronize_chan_irq(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + unsigned long *mask; + int i; + + /* + * A shared handler may retain this channel's status across quiesce. + * With nr_irqs == 1, it scans both directions even if routing and + * delegation are direction-wide. Drain it before allowing a routing change. + */ + for (i = 0; i < dw->nr_irqs; i++) { + mask = chan->dir == EDMA_DIR_WRITE ? dw->irq[i].wr_mask : + dw->irq[i].rd_mask; + if (!test_bit(chan->id, mask)) + continue; + + synchronize_irq(dw->chip->ops->irq_vector(dw->chip->dev, i)); + return; + } +} + static void dw_edma_device_synchronize(struct dma_chan *dchan) { struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + bool remote; + + /* + * irq_mode is fixed after initial configuration. The free path + * restores it only after synchronization. + */ + remote = chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL && + chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE; + + /* + * Peer-driven transfers bypass local descriptor tracking, so quiesce + * the hardware explicitly. + */ + if (remote && dw_edma_core_ch_quiesce(chan)) + dev_warn(chan->dw->chip->dev, + "failed to quiesce remote-routed %s channel %u\n", + chan->dir == EDMA_DIR_WRITE ? "write" : "read", + chan->id); dw_edma_wait_termination(dchan); + if (remote) + dw_edma_synchronize_chan_irq(chan); cancel_work_sync(&chan->irq_work); atomic_set(&chan->irq_pending, 0); vchan_synchronize(&chan->vc); @@ -907,8 +1102,10 @@ static void dw_edma_free_chan_resources(struct dma_chan *dchan) dw_edma_device_terminate_all(dchan); dw_edma_device_synchronize(dchan); - scoped_guard(spinlock_irqsave, &chan->vc.lock) + scoped_guard(spinlock_irqsave, &chan->vc.lock) { chan->configured = false; + chan->irq_mode = dw_edma_get_default_irq_mode(chan); + } vchan_free_chan_resources(&chan->vc); } @@ -954,6 +1151,13 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) else chan->ll_region = chip->ll_region_rd[chan->id]; + if (!chip->cfg_non_ll && chan->ll_region.sz < 3 * EDMA_LL_SZ) { + dev_err(dev, + "channel %s[%u]: LL region has fewer than 2 data entries\n", + str_write_read(chan->dir == EDMA_DIR_WRITE), + chan->id); + return -EINVAL; + } chan->ll_max = chan->ll_region.sz / EDMA_LL_SZ - 1; dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", @@ -988,6 +1192,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) &dw->chip->dt_region_rd[chan->id]; vchan_init(&chan->vc, dma); + dmaengine_set_static_chan_id(&chan->vc.chan, i); dw_edma_core_ch_config(chan); } @@ -1033,12 +1238,23 @@ static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) } } +static void dw_edma_compose_msi(int irq, struct msi_msg *msi) +{ + struct msi_desc *desc = irq_get_msi_desc(irq); + + if (!desc) + return; + + get_cached_msi_msg(irq, msi); + if (dev_is_pci(desc->dev) && !desc->pci.msi_attrib.is_msix) + msi->data += irq - desc->irq; +} + static int dw_edma_irq_request(struct dw_edma *dw, u32 *wr_alloc, u32 *rd_alloc) { struct dw_edma_chip *chip = dw->chip; struct device *dev = dw->chip->dev; - struct msi_desc *msi_desc; int i, err = 0; u32 ch_cnt; int irq; @@ -1063,8 +1279,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, return err; } - if (irq_get_msi_desc(irq)) - get_cached_msi_msg(irq, &dw->irq[0].msi); + dw_edma_compose_msi(irq, &dw->irq[0].msi); dw->nr_irqs = 1; } else { @@ -1087,12 +1302,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, &dw->irq[i]); if (err) goto err_irq_free; - msi_desc = irq_get_msi_desc(irq); - if (msi_desc) { - get_cached_msi_msg(irq, &dw->irq[i].msi); - if (!msi_desc->pci.msi_attrib.is_msix) - dw->irq[i].msi.data = dw->irq[0].msi.data + i; - } + dw_edma_compose_msi(irq, &dw->irq[i].msi); } dw->nr_irqs = i; diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index f6a5ad317567..76fd5a41a707 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -41,6 +41,13 @@ enum dw_edma_xfer_type { EDMA_XFER_INTERLEAVED }; +enum dw_edma_irq_event { + DW_EDMA_IRQ_DONE = BIT(0), + DW_EDMA_IRQ_PROGRESS = BIT(1), + DW_EDMA_IRQ_STOP = BIT(2), + DW_EDMA_IRQ_ABORT = BIT(3), +}; + struct dw_edma_chan; struct dw_edma_chunk; @@ -60,7 +67,6 @@ struct dw_edma_desc { size_t done_burst; size_t start_burst; - u8 cb; size_t nburst; struct dw_edma_burst burst[] __counted_by(nburst); }; @@ -72,8 +78,32 @@ struct dw_edma_chan { enum dw_edma_dir dir; u8 func_no; - u32 ll_max; + /* + * New LL entries are appended at ll_head. Entries between ll_done + * and ll_head, modulo the LL ring, are owned by DMA; the rest are + * owned by software. + * + * software-owned DMA-owned software-owned + * +---------------+-------------------+---------------+ + * ^ ^ ^ + * 0 ll_done ll_head + * + * The link entry points back to the region start. ll_head == ll_done + * means all entries are software-owned and previous DMA work is + * done. + * + * Software always keeps at least one free entry, so the ring is + * never completely DMA-owned. That keeps a hardware-reported physical + * LL index unique within the current ll_done..ll_head producer window. + */ + u32 ll_head; + u32 ll_done; + + u32 ll_max; /* Data entries */ struct dw_edma_region ll_region; /* Linked list */ + bool ll_valid; /* LL context programmed */ + + bool cb; struct msi_msg msi; @@ -124,7 +154,8 @@ struct dw_edma { const struct dw_edma_core_ops *core; }; -typedef void (*dw_edma_handler_t)(struct dw_edma_chan *); +typedef void (*dw_edma_handler_t)(struct dw_edma_chan *chan, + unsigned int events); struct dw_edma_core_ops { void (*off)(struct dw_edma *dw); @@ -133,11 +164,14 @@ struct dw_edma_core_ops { u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir); enum dma_status (*ch_status)(struct dw_edma_chan *chan); irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, - dw_edma_handler_t done, dw_edma_handler_t abort); + dw_edma_handler_t handler); void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child); void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst, u32 idx, bool cb, bool irq); void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr); + void (*ll_clear)(struct dw_edma_chan *chan, u32 idx); + int (*ll_cur_idx)(struct dw_edma_chan *chan); + void (*ll_irq_clear)(struct dw_edma_chan *chan); void (*ch_doorbell)(struct dw_edma_chan *chan); void (*ch_enable)(struct dw_edma_chan *chan); void (*ch_config)(struct dw_edma_chan *chan); @@ -181,6 +215,15 @@ struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan *dchan) return vc2dw_edma_chan(to_virt_chan(dchan)); } +/* + * Return the current LL entry index. A negative value means that the channel + * context is not initialized or was lost after a link reset. + */ +static inline int dw_edma_core_ll_cur_idx(struct dw_edma_chan *chan) +{ + return chan->dw->core->ll_cur_idx(chan); +} + static inline u64 dw_edma_core_get_ll_paddr(struct dw_edma_chan *chan) { if (chan->dir == EDMA_DIR_WRITE) @@ -221,9 +264,9 @@ enum dma_status dw_edma_core_ch_status(struct dw_edma_chan *chan) static inline irqreturn_t dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, - dw_edma_handler_t done, dw_edma_handler_t abort) + dw_edma_handler_t handler) { - return dw_irq->dw->core->handle_int(dw_irq, dir, done, abort); + return dw_irq->dw->core->handle_int(dw_irq, dir, handler); } static inline @@ -245,7 +288,17 @@ dw_edma_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr) chan->dw->core->ll_link(chan, idx, cb, addr); } -static inline void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan) +static inline void dw_edma_core_ll_clear(struct dw_edma_chan *chan, u32 idx) +{ + chan->dw->core->ll_clear(chan, idx); +} + +static inline void dw_edma_core_ll_irq_clear(struct dw_edma_chan *chan) +{ + chan->dw->core->ll_irq_clear(chan); +} + +static inline void dw_edma_core_do_ch_doorbell(struct dw_edma_chan *chan) { chan->dw->core->ch_doorbell(chan); } diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 9f237ba916de..fab4faf68b77 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -33,6 +33,8 @@ #define DW_PCIE_XILINX_MDB_VSEC_ID 0x20 #define DW_PCIE_XILINX_MDB_VSEC_DMA_BAR GENMASK(10, 8) #define DW_PCIE_XILINX_MDB_VSEC_DMA_MAP GENMASK(2, 0) +/* AMD CPM6 (Xilinx) supported cap */ +#define DW_PCIE_XILINX_CPM6_VSEC_CH_SEP GENMASK(18, 16) #define DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH GENMASK(9, 0) #define DW_PCIE_XILINX_MDB_VSEC_DMA_RD_CH GENMASK(25, 16) @@ -76,6 +78,7 @@ struct dw_edma_pcie_data { u16 rd_ch_cnt; u64 devmem_phys_off; bool cfg_non_ll; + u32 ch_space_sz; }; struct dw_edma_pcie_match_data { @@ -130,6 +133,7 @@ static const struct dw_edma_pcie_data snps_edda_data = { .irqs = 1, .wr_ch_cnt = 2, .rd_ch_cnt = 2, + .ch_space_sz = 256, }; static const struct dw_edma_pcie_data xilinx_mdb_data = { @@ -143,6 +147,7 @@ static const struct dw_edma_pcie_data xilinx_mdb_data = { .irqs = 1, .wr_ch_cnt = 8, .rd_ch_cnt = 8, + .ch_space_sz = 256, }; static const struct dw_edma_pcie_data xilinx_cpm6_dma_data = { @@ -156,6 +161,7 @@ static const struct dw_edma_pcie_data xilinx_cpm6_dma_data = { .irqs = 1, .wr_ch_cnt = 8, .rd_ch_cnt = 8, + .ch_space_sz = 512, }; static void dw_edma_set_chan_region_offset(struct dw_edma_pcie_data *pdata, @@ -208,6 +214,13 @@ static int dw_edma_pcie_irq_vector(struct device *dev, unsigned int nr) return pci_irq_vector(to_pci_dev(dev), nr); } +static u32 dw_edma_get_ch_space_sz(u32 val) +{ + if (val > 0 && val <= 7) + return 256 << val; + return 256; +} + static u64 dw_edma_pcie_address(struct device *dev, phys_addr_t cpu_addr) { struct pci_dev *pdev = to_pci_dev(dev); @@ -298,6 +311,10 @@ static void dw_edma_pcie_get_xilinx_dma_data(struct pci_dev *pdev, pdata->mf = map; pdata->rg.bar = FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_BAR, val); + if (pdev->device == PCI_DEVICE_ID_XILINX_B00F) + pdata->ch_space_sz = dw_edma_get_ch_space_sz + (FIELD_GET(DW_PCIE_XILINX_CPM6_VSEC_CH_SEP, val)); + pci_read_config_dword(pdev, vsec + 0xc, &val); pdata->wr_ch_cnt = min(pdata->wr_ch_cnt, FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH, val)); @@ -477,6 +494,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->nr_irqs = nr_irqs; chip->ops = match->plat_ops; chip->cfg_non_ll = dma_data->cfg_non_ll; + chip->ch_space_sz = dma_data->ch_space_sz; chip->ll_wr_cnt = dma_data->wr_ch_cnt; chip->ll_rd_cnt = dma_data->rd_ch_cnt; diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index abc1bd4c0be2..c053cb4c2844 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -336,7 +336,7 @@ static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan) static irqreturn_t dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, - dw_edma_handler_t done, dw_edma_handler_t abort) + dw_edma_handler_t handler) { struct dw_edma *dw = dw_irq->dw; unsigned long total, pos, val; @@ -375,7 +375,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, continue; dw_edma_v0_core_clear_done_int(chan); - done(chan); + handler(chan, DW_EDMA_IRQ_DONE); ret = IRQ_HANDLED; } @@ -389,7 +389,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, continue; dw_edma_v0_core_clear_abort_int(chan); - abort(chan); + handler(chan, DW_EDMA_IRQ_ABORT); ret = IRQ_HANDLED; } @@ -480,20 +480,6 @@ static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan) upper_32_bits(chan->ll_region.paddr)); } -static void dw_edma_v0_sync_ll_data(struct dw_edma_chan *chan) -{ - /* - * In case of remote eDMA engine setup, the DW PCIe RP/EP internal - * configuration registers and application memory are normally accessed - * over different buses. Ensure LL-data reaches the memory before the - * doorbell register is toggled by issuing the dummy-read from the remote - * LL memory in a hope that the MRd TLP will return only after the - * last MWr TLP is completed - */ - if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - readl(chan->ll_region.vaddr.io); -} - static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan) { struct dw_edma *dw = chan->dw; @@ -605,17 +591,49 @@ dw_edma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr) dw_edma_v0_write_ll_link(chan, idx, control, addr); } +static void dw_edma_v0_core_ll_clear(struct dw_edma_chan *chan, u32 idx) +{ + ptrdiff_t ofs = idx * sizeof(struct dw_edma_v0_lli); + + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_edma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs; + + lli->control = 0; + } else { + struct dw_edma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs; + + writel(0, &lli->control); + } +} + static void dw_edma_v0_core_ch_doorbell(struct dw_edma_chan *chan) { struct dw_edma *dw = chan->dw; - dw_edma_v0_sync_ll_data(chan); - /* Doorbell */ SET_RW_32(dw, chan->dir, doorbell, FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); } +static int dw_edma_v0_core_ll_cur_idx(struct dw_edma_chan *chan) +{ + u32 base, val; + + val = GET_CH_32(chan->dw, chan->dir, chan->id, llp.lsb); + base = lower_32_bits(dw_edma_core_get_ll_paddr(chan)); + + /* + * LL regions stay within one 4 GiB address window. Reject an all-ones + * MMIO value. If the low word is zero, use the high word to distinguish + * a nonzero boundary address from an unprogrammed all-zero context. + */ + if (val == U32_MAX || + (!val && !GET_CH_32(chan->dw, chan->dir, chan->id, llp.msb))) + return -EINVAL; + + return (val - base) / EDMA_LL_SZ; +} + /* eDMA debugfs callbacks */ static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw) { @@ -650,6 +668,9 @@ static const struct dw_edma_core_ops dw_edma_v0_core = { .handle_int = dw_edma_v0_core_handle_int, .ll_data = dw_edma_v0_core_ll_data, .ll_link = dw_edma_v0_core_ll_link, + .ll_clear = dw_edma_v0_core_ll_clear, + .ll_cur_idx = dw_edma_v0_core_ll_cur_idx, + .ll_irq_clear = dw_edma_v0_core_clear_done_int, .ch_doorbell = dw_edma_v0_core_ch_doorbell, .ch_enable = dw_edma_v0_core_ch_enable, .ch_config = dw_edma_v0_core_ch_config, diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 36ee72efcd31..d42cb4ad49a2 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -23,18 +23,23 @@ enum dw_hdma_control { DW_HDMA_V0_LLE = BIT(9), }; -static inline struct dw_hdma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) -{ - return dw->chip->reg_base; -} - static inline struct dw_hdma_v0_ch_regs __iomem * __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) { - if (dir == EDMA_DIR_WRITE) - return &(__dw_regs(dw)->ch[ch].wr); - else - return &(__dw_regs(dw)->ch[ch].rd); + u32 ch_base; + + /* + * For Write, the channel register index starts at + * wr_base(ch_idx) = (2 * ch_idx) * ch_space_sz + * + * For Read channel, + * rd_base(ch_idx) = (2 * ch_idx + 1) * ch_space_sz + */ + ch_base = 2 * ch; + if (dir == EDMA_DIR_READ) + ch_base += 1; + + return dw->chip->reg_base + (ch_base * dw->chip->ch_space_sz); } #define SET_CH_32(dw, dir, ch, name, value) \ @@ -174,7 +179,7 @@ static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan) static irqreturn_t dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, - dw_edma_handler_t done, dw_edma_handler_t abort) + dw_edma_handler_t handler) { struct dw_edma *dw = dw_irq->dw; unsigned long total, pos, val; @@ -200,14 +205,14 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, val = dw_hdma_v0_core_status_int(chan); if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) { dw_hdma_v0_core_clear_done_int(chan); - done(chan); + handler(chan, DW_EDMA_IRQ_STOP); ret = IRQ_HANDLED; } if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) { dw_hdma_v0_core_clear_abort_int(chan); - abort(chan); + handler(chan, DW_EDMA_IRQ_ABORT); ret = IRQ_HANDLED; } @@ -282,20 +287,6 @@ static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan) HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); } -static void dw_hdma_v0_sync_ll_data(struct dw_edma_chan *chan) -{ - /* - * In case of remote HDMA engine setup, the DW PCIe RP/EP internal - * configuration registers and application memory are normally accessed - * over different buses. Ensure LL-data reaches the memory before the - * doorbell register is toggled by issuing the dummy-read from the remote - * LL memory in a hope that the MRd TLP will return only after the - * last MWr TLP is completed - */ - if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - readl(chan->ll_region.vaddr.io); -} - static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chan *chan, struct dw_edma_burst *child) { @@ -374,16 +365,48 @@ dw_hdma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr) dw_hdma_v0_write_ll_link(chan, idx, control, addr); } +static void dw_hdma_v0_core_ll_clear(struct dw_edma_chan *chan, u32 idx) +{ + ptrdiff_t ofs = idx * sizeof(struct dw_hdma_v0_lli); + + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_hdma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs; + + lli->control = 0; + } else { + struct dw_hdma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs; + + writel(0, &lli->control); + } +} + static void dw_hdma_v0_core_ch_doorbell(struct dw_edma_chan *chan) { struct dw_edma *dw = chan->dw; - dw_hdma_v0_sync_ll_data(chan); - /* Doorbell */ SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); } +static int dw_hdma_v0_core_ll_cur_idx(struct dw_edma_chan *chan) +{ + u32 base, val; + + val = GET_CH_32(chan->dw, chan->dir, chan->id, llp.lsb); + base = lower_32_bits(dw_edma_core_get_ll_paddr(chan)); + + /* + * LL regions stay within one 4 GiB address window. Reject an all-ones + * MMIO value. If the low word is zero, use the high word to distinguish + * a nonzero boundary address from an unprogrammed all-zero context. + */ + if (val == U32_MAX || + (!val && !GET_CH_32(chan->dw, chan->dir, chan->id, llp.msb))) + return -EINVAL; + + return (val - base) / EDMA_LL_SZ; +} + /* HDMA debugfs callbacks */ static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw) { @@ -406,6 +429,9 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = { .non_ll_start = dw_hdma_v0_core_non_ll_start, .ll_data = dw_hdma_v0_core_ll_data, .ll_link = dw_hdma_v0_core_ll_link, + .ll_clear = dw_hdma_v0_core_ll_clear, + .ll_cur_idx = dw_hdma_v0_core_ll_cur_idx, + .ll_irq_clear = dw_hdma_v0_core_clear_done_int, .ch_doorbell = dw_hdma_v0_core_ch_doorbell, .ch_enable = dw_hdma_v0_core_ch_enable, .ch_config = dw_hdma_v0_core_ch_config, diff --git a/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c index dcdc57fe976c..3fa16e045a58 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c @@ -13,22 +13,17 @@ #include "dw-hdma-v0-regs.h" #include "dw-edma-core.h" -#define REGS_ADDR(dw, name) \ - ({ \ - struct dw_hdma_v0_regs __iomem *__regs = (dw)->chip->reg_base; \ - \ - (void __iomem *)&__regs->name; \ - }) - #define REGS_CH_ADDR(dw, name, _dir, _ch) \ ({ \ - struct dw_hdma_v0_ch_regs __iomem *__ch_regs; \ + struct dw_hdma_v0_ch_regs __iomem *__ch_regs; \ + off_t __off = (dw)->chip->ch_space_sz; \ \ - if (_dir == EDMA_DIR_READ) \ - __ch_regs = REGS_ADDR(dw, ch[_ch].rd); \ + if ((_dir) == EDMA_DIR_READ) \ + __off *= (2 * (_ch) + 1); \ else \ - __ch_regs = REGS_ADDR(dw, ch[_ch].wr); \ + __off *= (2 * (_ch)); \ \ + __ch_regs = ((dw)->chip->reg_base + __off); \ (void __iomem *)&__ch_regs->name; \ }) diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h index 2124c162a62f..b88226cdd791 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h +++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h @@ -85,16 +85,6 @@ struct dw_hdma_v0_ch_regs { }; } msi_abort; u32 msi_msgdata; /* 0x00a8 */ - u32 padding_2[21]; /* 0x00ac..0x00fc */ -} __packed; - -struct dw_hdma_v0_ch { - struct dw_hdma_v0_ch_regs wr; /* 0x0000 */ - struct dw_hdma_v0_ch_regs rd; /* 0x0100 */ -} __packed; - -struct dw_hdma_v0_regs { - struct dw_hdma_v0_ch ch[HDMA_V0_MAX_NR_CH]; /* 0x0000..0x0fa8 */ } __packed; struct dw_hdma_v0_lli { diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index dd75f97a33b3..575b7bb6af9a 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -41,11 +41,6 @@ /*----------------------------------------------------------------------*/ -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc) { return to_dw_desc(dwc->active_list.next); @@ -69,7 +64,7 @@ static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx) list_add_tail(&desc->desc_node, &dwc->queue); spin_unlock_irqrestore(&dwc->lock, flags); - dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", + dev_vdbg(dmaengine_chan_dev(tx->chan), "%s: queued %u\n", __func__, desc->txd.cookie); return cookie; @@ -127,7 +122,7 @@ static void dwc_initialize(struct dw_dma_chan *dwc) static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc) { - dev_err(chan2dev(&dwc->chan), + dev_err(dmaengine_chan_dev(&dwc->chan), " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n", channel_readl(dwc, SAR), channel_readl(dwc, DAR), @@ -177,7 +172,7 @@ static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first) /* ASSERT: channel is idle */ if (dma_readl(dw, CH_EN) & dwc->mask) { - dev_err(chan2dev(&dwc->chan), + dev_err(dmaengine_chan_dev(&dwc->chan), "%s: BUG: Attempted to start non-idle channel\n", __func__); dwc_dump_chan_regs(dwc); @@ -190,7 +185,7 @@ static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first) was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags); if (was_soft_llp) { - dev_err(chan2dev(&dwc->chan), + dev_err(dmaengine_chan_dev(&dwc->chan), "BUG: Attempted to start new LLP transfer inside ongoing one\n"); return; } @@ -223,7 +218,7 @@ static void dwc_dostart_first_queued(struct dw_dma_chan *dwc) list_move(dwc->queue.next, &dwc->active_list); desc = dwc_first_active(dwc); - dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie); + dev_vdbg(dmaengine_chan_dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie); dwc_dostart(dwc, desc); } @@ -238,7 +233,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc, unsigned long flags; struct dmaengine_desc_callback cb; - dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie); + dev_vdbg(dmaengine_chan_dev(&dwc->chan), "descriptor %u complete\n", txd->cookie); spin_lock_irqsave(&dwc->lock, flags); dma_cookie_complete(txd); @@ -265,7 +260,7 @@ static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc) spin_lock_irqsave(&dwc->lock, flags); if (dma_readl(dw, CH_EN) & dwc->mask) { - dev_err(chan2dev(&dwc->chan), + dev_err(dmaengine_chan_dev(&dwc->chan), "BUG: XFER bit set, but channel not idle!\n"); /* Try to continue after resetting the channel... */ @@ -353,12 +348,12 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc) } if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) { - dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__); + dev_vdbg(dmaengine_chan_dev(&dwc->chan), "%s: soft LLP mode\n", __func__); spin_unlock_irqrestore(&dwc->lock, flags); return; } - dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp); + dev_vdbg(dmaengine_chan_dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp); list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) { /* Initial residue value */ @@ -398,7 +393,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc) spin_lock_irqsave(&dwc->lock, flags); } - dev_err(chan2dev(&dwc->chan), + dev_err(dmaengine_chan_dev(&dwc->chan), "BUG: All descriptors done, but channel not idle!\n"); /* Try to continue after resetting the channel... */ @@ -410,7 +405,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc) static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc) { - dev_crit(chan2dev(&dwc->chan), " desc: s0x%x d0x%x l0x%x c0x%x:%x\n", + dev_crit(dmaengine_chan_dev(&dwc->chan), " desc: s0x%x d0x%x l0x%x c0x%x:%x\n", lli_read(desc, sar), lli_read(desc, dar), lli_read(desc, llp), @@ -449,7 +444,7 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc) * controller flagged an error instead of scribbling over * random memory locations. */ - dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n" + dev_WARN(dmaengine_chan_dev(&dwc->chan), "Bad descriptor submitted for DMA!\n" " cookie: %d\n", bad_desc->txd.cookie); dwc_dump_lli(dwc, bad_desc); list_for_each_entry(child, &bad_desc->tx_list, desc_node) @@ -552,12 +547,12 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, u32 ctllo, ctlhi; u8 lms = DWC_LLP_LMS(m_master); - dev_vdbg(chan2dev(chan), + dev_vdbg(dmaengine_chan_dev(chan), "%s: d%pad s%pad l0x%zx f0x%lx\n", __func__, &dest, &src, len, flags); if (unlikely(!len)) { - dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: length is zero!\n", __func__); return NULL; } @@ -630,7 +625,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, struct scatterlist *sg; size_t total_len = 0; - dev_vdbg(chan2dev(chan), "%s\n", __func__); + dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__); if (unlikely(!is_slave_direction(direction) || !sg_len)) return NULL; @@ -754,7 +749,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return &first->txd; err_desc_get: - dev_err(chan2dev(chan), + dev_err(dmaengine_chan_dev(chan), "not enough descriptors available. Direction %d\n", direction); dwc_desc_put(dwc, first); return NULL; @@ -1063,11 +1058,11 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan) struct dw_dma_chan *dwc = to_dw_dma_chan(chan); struct dw_dma *dw = to_dw_dma(chan->device); - dev_vdbg(chan2dev(chan), "%s\n", __func__); + dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__); /* ASSERT: channel is idle */ if (dma_readl(dw, CH_EN) & dwc->mask) { - dev_dbg(chan2dev(chan), "DMA channel not idle?\n"); + dev_dbg(dmaengine_chan_dev(chan), "DMA channel not idle?\n"); return -EIO; } @@ -1083,7 +1078,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan) * We need controller-specific data to set up slave transfers. */ if (chan->private && !dw_dma_filter(chan, chan->private)) { - dev_warn(chan2dev(chan), "Wrong controller-specific data\n"); + dev_warn(dmaengine_chan_dev(chan), "Wrong controller-specific data\n"); return -EINVAL; } @@ -1101,7 +1096,7 @@ static void dwc_free_chan_resources(struct dma_chan *chan) struct dw_dma *dw = to_dw_dma(chan->device); unsigned long flags; - dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__, + dev_dbg(dmaengine_chan_dev(chan), "%s: descs allocated=%u\n", __func__, dwc->descs_allocated); /* ASSERT: channel is idle */ @@ -1126,7 +1121,7 @@ static void dwc_free_chan_resources(struct dma_chan *chan) if (!dw->in_use) do_dw_dma_off(dw); - dev_vdbg(chan2dev(chan), "%s: done\n", __func__); + dev_vdbg(dmaengine_chan_dev(chan), "%s: done\n", __func__); } static void dwc_caps(struct dma_chan *chan, struct dma_slave_caps *caps) diff --git a/drivers/dma/dw/dw.c b/drivers/dma/dw/dw.c index 6766142884b6..e56974e7d4af 100644 --- a/drivers/dma/dw/dw.c +++ b/drivers/dma/dw/dw.c @@ -9,6 +9,7 @@ #include #include +#include "../dmaengine.h" #include "internal.h" static void dw_dma_initialize_chan(struct dw_dma_chan *dwc) diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c index c63fa52036d7..5c1c97eb28a4 100644 --- a/drivers/dma/dw/platform.c +++ b/drivers/dma/dw/platform.c @@ -141,14 +141,14 @@ MODULE_DEVICE_TABLE(of, dw_dma_of_id_table); #ifdef CONFIG_ACPI static const struct acpi_device_id dw_dma_acpi_id_table[] = { - { "INTL9C60", (kernel_ulong_t)&dw_dma_chip_pdata }, - { "80862286", (kernel_ulong_t)&dw_dma_chip_pdata }, - { "808622C0", (kernel_ulong_t)&dw_dma_chip_pdata }, + { .id = "INTL9C60", .driver_data = (kernel_ulong_t)&dw_dma_chip_pdata }, + { .id = "80862286", .driver_data = (kernel_ulong_t)&dw_dma_chip_pdata }, + { .id = "808622C0", .driver_data = (kernel_ulong_t)&dw_dma_chip_pdata }, /* Elkhart Lake iDMA 32-bit (PSE DMA) */ - { "80864BB4", (kernel_ulong_t)&xbar_chip_pdata }, - { "80864BB5", (kernel_ulong_t)&xbar_chip_pdata }, - { "80864BB6", (kernel_ulong_t)&xbar_chip_pdata }, + { .id = "80864BB4", .driver_data = (kernel_ulong_t)&xbar_chip_pdata }, + { .id = "80864BB5", .driver_data = (kernel_ulong_t)&xbar_chip_pdata }, + { .id = "80864BB6", .driver_data = (kernel_ulong_t)&xbar_chip_pdata }, { } }; diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c index 311e55a97ba9..cbfe74fc8d71 100644 --- a/drivers/dma/ep93xx_dma.c +++ b/drivers/dma/ep93xx_dma.c @@ -250,11 +250,6 @@ struct ep93xx_edma_data { size_t num_channels; }; -static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac) -{ - return &edmac->chan.dev->device; -} - static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan) { return container_of(chan, struct ep93xx_dma_chan, chan); @@ -415,7 +410,7 @@ static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac) m2p_set_control(edmac, 0); while (m2p_channel_state(edmac) != M2P_STATE_IDLE) - dev_warn(chan2dev(edmac), "M2P: Not yet IDLE\n"); + dev_warn(dmaengine_chan_dev(&edmac->chan), "M2P: Not yet IDLE\n"); } static void m2p_fill_desc(struct ep93xx_dma_chan *edmac) @@ -425,7 +420,7 @@ static void m2p_fill_desc(struct ep93xx_dma_chan *edmac) desc = ep93xx_dma_get_active(edmac); if (!desc) { - dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n"); + dev_warn(dmaengine_chan_dev(&edmac->chan), "M2P: empty descriptor list\n"); return; } @@ -479,7 +474,7 @@ static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac) * Revisit this when there is a mechanism to report back the * errors. */ - dev_err(chan2dev(edmac), + dev_err(dmaengine_chan_dev(&edmac->chan), "DMA transfer failed! Details:\n" "\tcookie : %d\n" "\tsrc_addr : 0x%08x\n" @@ -586,7 +581,7 @@ static void m2m_fill_desc(struct ep93xx_dma_chan *edmac) desc = ep93xx_dma_get_active(edmac); if (!desc) { - dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n"); + dev_warn(dmaengine_chan_dev(&edmac->chan), "M2M: empty descriptor list\n"); return; } @@ -849,7 +844,7 @@ static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id) desc = ep93xx_dma_get_active(edmac); if (!desc) { - dev_warn(chan2dev(edmac), + dev_warn(dmaengine_chan_dev(&edmac->chan), "got interrupt while active list is empty\n"); spin_unlock(&edmac->lock); return IRQ_NONE; @@ -867,7 +862,7 @@ static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id) break; default: - dev_warn(chan2dev(edmac), "unknown interrupt!\n"); + dev_warn(dmaengine_chan_dev(&edmac->chan), "unknown interrupt!\n"); ret = IRQ_NONE; break; } @@ -967,7 +962,7 @@ static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan) desc = kzalloc_obj(*desc); if (!desc) { - dev_warn(chan2dev(edmac), "not enough descriptors\n"); + dev_warn(dmaengine_chan_dev(&edmac->chan), "not enough descriptors\n"); break; } @@ -1044,7 +1039,7 @@ ep93xx_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, for (offset = 0; offset < len; offset += bytes) { desc = ep93xx_dma_desc_get(edmac); if (!desc) { - dev_warn(chan2dev(edmac), "couldn't get descriptor\n"); + dev_warn(dmaengine_chan_dev(&edmac->chan), "couldn't get descriptor\n"); goto fail; } @@ -1091,13 +1086,13 @@ ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, int i; if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) { - dev_warn(chan2dev(edmac), + dev_warn(dmaengine_chan_dev(&edmac->chan), "channel was configured with different direction\n"); return NULL; } if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) { - dev_warn(chan2dev(edmac), + dev_warn(dmaengine_chan_dev(&edmac->chan), "channel is already used for cyclic transfers\n"); return NULL; } @@ -1109,14 +1104,14 @@ ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, size_t len = sg_dma_len(sg); if (len > DMA_MAX_CHAN_BYTES) { - dev_warn(chan2dev(edmac), "too big transfer size %zu\n", + dev_warn(dmaengine_chan_dev(&edmac->chan), "too big transfer size %zu\n", len); goto fail; } desc = ep93xx_dma_desc_get(edmac); if (!desc) { - dev_warn(chan2dev(edmac), "couldn't get descriptor\n"); + dev_warn(dmaengine_chan_dev(&edmac->chan), "couldn't get descriptor\n"); goto fail; } @@ -1172,19 +1167,19 @@ ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr, size_t offset = 0; if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) { - dev_warn(chan2dev(edmac), + dev_warn(dmaengine_chan_dev(&edmac->chan), "channel was configured with different direction\n"); return NULL; } if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) { - dev_warn(chan2dev(edmac), + dev_warn(dmaengine_chan_dev(&edmac->chan), "channel is already used for cyclic transfers\n"); return NULL; } if (period_len > DMA_MAX_CHAN_BYTES) { - dev_warn(chan2dev(edmac), "too big period length %zu\n", + dev_warn(dmaengine_chan_dev(&edmac->chan), "too big period length %zu\n", period_len); return NULL; } @@ -1196,7 +1191,7 @@ ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr, for (offset = 0; offset < buf_len; offset += period_len) { desc = ep93xx_dma_desc_get(edmac); if (!desc) { - dev_warn(chan2dev(edmac), "couldn't get descriptor\n"); + dev_warn(dmaengine_chan_dev(&edmac->chan), "couldn't get descriptor\n"); goto fail; } diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index bb7531c456df..513b0f4e5be2 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -3,6 +3,7 @@ // Copyright (c) 2013-2014 Freescale Semiconductor, Inc // Copyright (c) 2017 Sysam, Angelo Dureghello +#include #include #include #include @@ -55,10 +56,36 @@ void fsl_edma_tx_chan_handler(struct fsl_edma_chan *fsl_chan) } if (!fsl_chan->edesc->iscyclic) { - list_del(&fsl_chan->edesc->vdesc.node); - vchan_cookie_complete(&fsl_chan->edesc->vdesc); + u16 csr = edma_read_tcdreg(fsl_chan, csr); + u8 link_sg_id = FIELD_GET(EDMA_TCD_CSR_LINKCH, csr); + struct virt_dma_desc *vdesc, *tmp; + + /* Channel is DONE when a TCD with D_REQ set completes */ + if (!(fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_SPLIT_REG) || + (edma_readl_chreg(fsl_chan, ch_csr) & EDMA_V3_CH_CSR_DONE)) { + fsl_chan->status = DMA_COMPLETE; + } + + list_for_each_entry_safe(vdesc, tmp, &fsl_chan->vchan.desc_issued, node) { + struct fsl_edma_desc *fsl_desc = to_fsl_edma_desc(vdesc); + bool id_match = (link_sg_id == fsl_desc->link_sg_id); + + /* + * If the transfer is still running, + * don't mark as complete the current descriptor + */ + if (id_match && fsl_chan->status != DMA_COMPLETE) + break; + + list_del(&vdesc->node); + vchan_cookie_complete(vdesc); + + if (id_match) + break; + } + fsl_chan->edesc = NULL; - fsl_chan->status = DMA_COMPLETE; + } else { vchan_cyclic_callback(&fsl_chan->edesc->vdesc); } @@ -673,6 +700,122 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); } +struct dma_async_tx_descriptor * +fsl_edma_prep_peripheral_dma_vec(struct dma_chan *chan, const struct dma_vec *vecs, + size_t nb, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + dma_addr_t src_addr, dst_addr, last_sg; + struct fsl_edma_desc *fsl_desc; + u16 soff, doff, iter; + u32 nbytes; + int i; + + if (!is_slave_direction(direction)) + return NULL; + + if (!fsl_edma_prep_slave_dma(fsl_chan, direction)) + return NULL; + + fsl_desc = fsl_edma_alloc_desc(fsl_chan, nb); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = flags & DMA_PREP_REPEAT; + fsl_desc->dirn = direction; + + if (direction == DMA_MEM_TO_DEV) { + if (!fsl_chan->cfg.src_addr_width) + fsl_chan->cfg.src_addr_width = fsl_chan->cfg.dst_addr_width; + fsl_chan->attr = + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); + nbytes = fsl_chan->cfg.dst_addr_width * fsl_chan->cfg.dst_maxburst; + } else { + if (!fsl_chan->cfg.dst_addr_width) + fsl_chan->cfg.dst_addr_width = fsl_chan->cfg.src_addr_width; + fsl_chan->attr = + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); + nbytes = fsl_chan->cfg.src_addr_width * fsl_chan->cfg.src_maxburst; + } + + for (i = 0; i < nb; i++) { + if (direction == DMA_MEM_TO_DEV) { + src_addr = vecs[i].addr; + dst_addr = fsl_chan->dma_dev_addr; + soff = fsl_chan->cfg.dst_addr_width; + doff = 0; + } else if (direction == DMA_DEV_TO_MEM) { + src_addr = fsl_chan->dma_dev_addr; + dst_addr = vecs[i].addr; + soff = 0; + doff = fsl_chan->cfg.src_addr_width; + } else { + /* DMA_DEV_TO_DEV */ + src_addr = fsl_chan->cfg.src_addr; + dst_addr = fsl_chan->cfg.dst_addr; + soff = 0; + doff = 0; + } + + /* + * Choose the suitable burst length if dma_vec length is not + * multiple of burst length so that the whole transfer length is + * multiple of minor loop(burst length). + */ + if (nbytes && vecs[i].len % nbytes) { + u32 width = (direction == DMA_DEV_TO_MEM) ? doff : soff; + u32 burst = (direction == DMA_DEV_TO_MEM) ? + fsl_chan->cfg.src_maxburst : + fsl_chan->cfg.dst_maxburst; + int j; + + for (j = burst; j > 1; j--) { + if (!(vecs[i].len % (j * width))) { + nbytes = j * width; + break; + } + } + /* Set burst size as 1 if there's no suitable one */ + if (j == 1) + nbytes = width; + } + + if (!nbytes || vecs[i].len / nbytes > FIELD_MAX(EDMA_TCD_ITER_MASK)) + goto err_free_desc; + + iter = vecs[i].len / nbytes; + if (i < nb - 1) { + last_sg = fsl_desc->tcd[(i + 1)].ptcd; + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->attr, soff, + nbytes, 0, iter, iter, doff, last_sg, + false, false, true); + } else { + if (fsl_desc->iscyclic) { + last_sg = fsl_desc->tcd[0].ptcd; + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->attr, soff, + nbytes, 0, iter, iter, doff, last_sg, + true, false, true); + } else { + last_sg = 0; + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->attr, soff, + nbytes, 0, iter, iter, doff, last_sg, + true, true, false); + } + } + } + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); + +err_free_desc: + fsl_edma_free_desc(&fsl_desc->vdesc); + return NULL; +} + struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, @@ -815,14 +958,99 @@ void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) if (!vdesc) return; fsl_chan->edesc = to_fsl_edma_desc(vdesc); - fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); - fsl_edma_enable_request(fsl_chan); - fsl_chan->status = DMA_IN_PROGRESS; + + if (fsl_chan->status != DMA_IN_PROGRESS) { + fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); + fsl_edma_enable_request(fsl_chan); + fsl_chan->status = DMA_IN_PROGRESS; + } +} + +static void fsl_edma_link_sg(struct fsl_edma_chan *fsl_chan, struct fsl_edma_desc *fsl_desc) +{ + u32 flags = fsl_edma_drvflags(fsl_chan); + struct fsl_edma_hw_tcd *last_tcd; + struct fsl_edma_desc *prev_desc; + struct virt_dma_desc *vdesc; + u16 last_csr; + + lockdep_assert_held(&fsl_chan->vchan.lock); + + if (!(flags & FSL_EDMA_DRV_CSR_LINKCH) || fsl_desc->iscyclic) + return; + + vdesc = list_last_entry_or_null(&fsl_chan->vchan.desc_issued, + struct virt_dma_desc, node); + if (!vdesc) + return; + + prev_desc = to_fsl_edma_desc(vdesc); + if (prev_desc->iscyclic) + return; + + last_tcd = prev_desc->tcd[prev_desc->n_tcds - 1].vtcd; + last_csr = fsl_edma_get_tcd_to_cpu(fsl_chan, last_tcd, csr); + + for (unsigned int i = 0; i < fsl_desc->n_tcds; i++) { + struct fsl_edma_hw_tcd *tcd = fsl_desc->tcd[i].vtcd; + + if (fsl_edma_get_tcd_to_cpu(fsl_chan, tcd, csr) & EDMA_TCD_CSR_E_LINK) + return; + } + + if (!(last_csr & EDMA_TCD_CSR_D_REQ) || + last_csr & EDMA_TCD_CSR_E_LINK) + return; + + /* Set a non-zero linked SG identifier to all TCD of the new descriptor */ + fsl_chan->link_sg_id++; + if (fsl_chan->link_sg_id > FIELD_MAX(EDMA_TCD_CSR_LINKCH)) + fsl_chan->link_sg_id = 1; + + fsl_desc->link_sg_id = fsl_chan->link_sg_id; + + for (unsigned int i = 0; i < fsl_desc->n_tcds; i++) { + struct fsl_edma_hw_tcd *tcd = fsl_desc->tcd[i].vtcd; + u16 csr = fsl_edma_get_tcd_to_cpu(fsl_chan, tcd, csr); + + csr |= FIELD_PREP(EDMA_TCD_CSR_LINKCH, fsl_chan->link_sg_id); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, csr, csr); + } + + /* + * Set DLAST_SGA before enabling E_SG in CSR: if the DMA engine + * only picks up the former, it updates DADDR at end of transfer, + * which is not reused. + */ + fsl_edma_set_tcd_to_le(fsl_chan, last_tcd, fsl_desc->tcd[0].ptcd, dlast_sga); + + dma_wmb(); + + last_csr &= ~EDMA_TCD_CSR_D_REQ; + last_csr |= EDMA_TCD_CSR_E_SG; + fsl_edma_set_tcd_to_le(fsl_chan, last_tcd, last_csr, csr); + + /* + * Update the registers if the engine already loaded the TCD, + * unless doing so requires clearing the DONE bit, as that could + * lead to lost completion. + */ + if (prev_desc == fsl_chan->edesc && + prev_desc->n_tcds == 1 && + !(flags & FSL_EDMA_DRV_CLEAR_DONE_E_SG)) { + edma_cp_tcd_to_reg(fsl_chan, last_tcd, dlast_sga); + edma_cp_tcd_to_reg(fsl_chan, last_tcd, csr); + } + + trace_edma_link_sg(fsl_chan, last_tcd); } void fsl_edma_issue_pending(struct dma_chan *chan) { struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct virt_dma_desc *vdesc, *tmp; + size_t issued_count = 0; + struct list_head *pos; unsigned long flags; spin_lock_irqsave(&fsl_chan->vchan.lock, flags); @@ -833,6 +1061,21 @@ void fsl_edma_issue_pending(struct dma_chan *chan) return; } + /* Count the issued desc up to the maximum number of linked SG id */ + list_for_each(pos, &fsl_chan->vchan.desc_issued) { + if (++issued_count > FIELD_MAX(EDMA_TCD_CSR_LINKCH)) + break; + } + + /* Link the SG descriptors with the available identifiers */ + list_for_each_entry_safe(vdesc, tmp, &fsl_chan->vchan.desc_submitted, node) { + if (++issued_count > FIELD_MAX(EDMA_TCD_CSR_LINKCH)) + break; + fsl_edma_link_sg(fsl_chan, to_fsl_edma_desc(vdesc)); + list_move_tail(&vdesc->node, &fsl_chan->vchan.desc_issued); + } + + /* Issue the rest of the descriptors unlinked */ if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) fsl_edma_xfer_desc(fsl_chan); diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h index 205a96489094..cbb077e98f55 100644 --- a/drivers/dma/fsl-edma-common.h +++ b/drivers/dma/fsl-edma-common.h @@ -42,6 +42,7 @@ #define EDMA_TCD_CSR_E_LINK BIT(5) #define EDMA_TCD_CSR_ACTIVE BIT(6) #define EDMA_TCD_CSR_DONE BIT(7) +#define EDMA_TCD_CSR_LINKCH GENMASK(12, 8) #define EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(x) ((x) & GENMASK(9, 0)) #define EDMA_V3_TCD_NBYTES_MLOFF(x) (x << 10) @@ -169,6 +170,7 @@ struct fsl_edma_chan { struct dma_slave_config cfg; u32 attr; bool is_sw; + u8 link_sg_id; struct dma_pool *tcd_pool; dma_addr_t dma_dev_addr; u32 dma_dev_size; @@ -199,6 +201,7 @@ struct fsl_edma_desc { struct virt_dma_desc vdesc; struct fsl_edma_chan *echan; bool iscyclic; + u8 link_sg_id; enum dma_transfer_direction dirn; unsigned int n_tcds; struct fsl_edma_sw_tcd tcd[]; @@ -225,6 +228,8 @@ struct fsl_edma_desc { #define FSL_EDMA_DRV_TCD64 BIT(15) /* All channel ERR IRQ share one IRQ line */ #define FSL_EDMA_DRV_ERRIRQ_SHARE BIT(16) +/* Major link channel in CSR used for linking SG descriptors */ +#define FSL_EDMA_DRV_CSR_LINKCH BIT(17) #define FSL_EDMA_DRV_EDMA3 (FSL_EDMA_DRV_SPLIT_REG | \ @@ -232,13 +237,15 @@ struct fsl_edma_desc { FSL_EDMA_DRV_DEV_TO_DEV | \ FSL_EDMA_DRV_ALIGN_64BYTE | \ FSL_EDMA_DRV_CLEAR_DONE_E_SG | \ - FSL_EDMA_DRV_CLEAR_DONE_E_LINK) + FSL_EDMA_DRV_CLEAR_DONE_E_LINK | \ + FSL_EDMA_DRV_CSR_LINKCH) #define FSL_EDMA_DRV_EDMA4 (FSL_EDMA_DRV_SPLIT_REG | \ FSL_EDMA_DRV_BUS_8BYTE | \ FSL_EDMA_DRV_DEV_TO_DEV | \ FSL_EDMA_DRV_ALIGN_64BYTE | \ - FSL_EDMA_DRV_CLEAR_DONE_E_LINK) + FSL_EDMA_DRV_CLEAR_DONE_E_LINK | \ + FSL_EDMA_DRV_CSR_LINKCH) struct fsl_edma_drvdata { u32 dmamuxs; /* only used before v3 */ @@ -496,6 +503,10 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, unsigned long flags); +struct dma_async_tx_descriptor *fsl_edma_prep_peripheral_dma_vec( + struct dma_chan *chan, const struct dma_vec *vecs, + size_t nb, enum dma_transfer_direction direction, + unsigned long flags); struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index d9fb717b5b53..630285c85482 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -326,13 +326,16 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1)) continue; + chan = dma_get_slave_channel(chan); + if (!chan) + return NULL; + fsl_chan->srcid = dma_spec->args[0]; fsl_chan->priority = dma_spec->args[1]; fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; - chan = dma_get_slave_channel(chan); chan->device->privatecnt++; return chan; } @@ -843,6 +846,8 @@ static int fsl_edma_probe(struct platform_device *pdev) fsl_edma->dma_dev.device_free_chan_resources = fsl_edma_free_chan_resources; fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status; + fsl_edma->dma_dev.device_prep_peripheral_dma_vec + = fsl_edma_prep_peripheral_dma_vec; fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg; fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic; fsl_edma->dma_dev.device_prep_dma_memcpy = fsl_edma_prep_memcpy; diff --git a/drivers/dma/fsl-edma-trace.h b/drivers/dma/fsl-edma-trace.h index 45d964a3726d..f4b0021d2bd0 100644 --- a/drivers/dma/fsl-edma-trace.h +++ b/drivers/dma/fsl-edma-trace.h @@ -121,6 +121,11 @@ DEFINE_EVENT(edma_log_tcd, edma_fill_tcd, TP_ARGS(chan, tcd) ); +DEFINE_EVENT(edma_log_tcd, edma_link_sg, + TP_PROTO(struct fsl_edma_chan *chan, void *tcd), + TP_ARGS(chan, tcd) +); + #endif /* this part must be outside header guard */ diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index 2d86f61105e5..9cc289f7129e 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -162,6 +162,7 @@ static void fsl_re_dequeue(struct tasklet_struct *t) struct fsl_re_hw_desc *hwdesc; unsigned long flags; unsigned int count, oub_count; + LIST_HEAD(completed); int found; fsl_re_cleanup_descs(re_chan); @@ -182,8 +183,7 @@ static void fsl_re_dequeue(struct tasklet_struct *t) } if (found) { - fsl_re_desc_done(desc); - list_move_tail(&desc->node, &re_chan->ack_q); + list_move_tail(&desc->node, &completed); } else { dev_err(re_chan->dev, "found hwdesc not in sw queue, discard it\n"); @@ -196,6 +196,17 @@ static void fsl_re_dequeue(struct tasklet_struct *t) FSL_RE_RMVD_JOB(1)); } spin_unlock_irqrestore(&re_chan->desc_lock, flags); + + /* Invoke the client callbacks outside the channel lock. The callback + * may submit new work which re-acquires desc_lock, so holding it here + * would deadlock. + */ + list_for_each_entry_safe(desc, _desc, &completed, node) { + fsl_re_desc_done(desc); + spin_lock_irqsave(&re_chan->desc_lock, flags); + list_move_tail(&desc->node, &re_chan->ack_q); + spin_unlock_irqrestore(&re_chan->desc_lock, flags); + } } /* Per Job Ring interrupt handler */ @@ -295,8 +306,8 @@ static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan, if (!desc) return NULL; - cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT, - &paddr); + cf = dma_pool_zalloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT, + &paddr); if (!cf) { kfree(desc); return NULL; @@ -350,7 +361,7 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq( cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT; cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; xor = desc->cdb_addr; - xor->cdb32 = cdb; + xor->cdb32 = cpu_to_be32(cdb); if (scf) { /* compute q = src0*coef0^src1*coef1^..., * is GF(8) mult */ @@ -470,7 +481,7 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq( cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; pq = desc->cdb_addr; - pq->cdb32 = cdb; + pq->cdb32 = cpu_to_be32(cdb); p = pq->gfm_q1; /* Init gfm_q1[] */ @@ -553,7 +564,7 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_memcpy( cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; move = desc->cdb_addr; - move->cdb32 = cdb; + move->cdb32 = cpu_to_be32(cdb); /* Filling frame 0 of CFD with move CDB */ cf = desc->cf_addr; @@ -584,8 +595,8 @@ static int fsl_re_alloc_chan_resources(struct dma_chan *chan) if (!desc) break; - cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL, - &paddr); + cf = dma_pool_zalloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL, + &paddr); if (!cf) { kfree(desc); break; @@ -603,18 +614,15 @@ static int fsl_re_alloc_chan_resources(struct dma_chan *chan) static void fsl_re_free_chan_resources(struct dma_chan *chan) { struct fsl_re_chan *re_chan; - struct fsl_re_desc *desc; + struct fsl_re_desc *desc, *_desc; re_chan = container_of(chan, struct fsl_re_chan, chan); - while (re_chan->alloc_count--) { - desc = list_first_entry(&re_chan->free_q, - struct fsl_re_desc, - node); - + list_for_each_entry_safe(desc, _desc, &re_chan->free_q, node) { list_del(&desc->node); dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr, desc->cf_paddr); kfree(desc); + re_chan->alloc_count--; } if (!list_empty(&re_chan->free_q)) diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index 22d62d958abd..99a878ed1444 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -681,7 +681,7 @@ static void fsldma_cleanup_descriptors(struct fsldma_chan *chan) /** * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel. - * @chan : Freescale DMA channel + * @dchan : Freescale DMA channel * * This function will create a dma pool for descriptor allocation. * @@ -738,7 +738,7 @@ static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan, /** * fsl_dma_free_chan_resources - Free all resources of the channel. - * @chan : Freescale DMA channel + * @dchan : Freescale DMA channel */ static void fsl_dma_free_chan_resources(struct dma_chan *dchan) { @@ -873,7 +873,7 @@ static int fsl_dma_device_config(struct dma_chan *dchan, /** * fsl_dma_memcpy_issue_pending - Issue the DMA start command - * @chan : Freescale DMA channel + * @dchan : Freescale DMA channel */ static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan) { @@ -886,7 +886,9 @@ static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan) /** * fsl_tx_status - Determine the DMA status - * @chan : Freescale DMA channel + * @dchan : Freescale DMA channel + * @cookie : DMA transaction identifier + * @txstate : DMA transaction state */ static enum dma_status fsl_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, @@ -1205,6 +1207,7 @@ static int fsl_dma_chan_probe(struct fsldma_device *fdev, static void fsl_dma_chan_remove(struct fsldma_chan *chan) { + tasklet_kill(&chan->tasklet); irq_dispose_mapping(chan->irq); list_del(&chan->common.device_node); iounmap(chan->regs); @@ -1216,8 +1219,13 @@ static int fsldma_of_probe(struct platform_device *op) struct fsldma_device *fdev; struct device_node *child; unsigned int i; + int irq; int err; + irq = platform_get_irq_optional(op, 0); + if (irq == -EPROBE_DEFER) + return irq; + fdev = kzalloc_obj(*fdev); if (!fdev) { err = -ENOMEM; @@ -1238,7 +1246,9 @@ static int fsldma_of_probe(struct platform_device *op) } /* map the channel IRQ if it exists, but don't hookup the handler yet */ - fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0); + fdev->irq = irq; + if (fdev->irq < 0) + fdev->irq = 0; dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask); dma_cap_set(DMA_SLAVE, fdev->common.cap_mask); @@ -1300,7 +1310,6 @@ static int fsldma_of_probe(struct platform_device *op) if (fdev->chan[i]) fsl_dma_chan_remove(fdev->chan[i]); } - irq_dispose_mapping(fdev->irq); iounmap(fdev->regs); out_free: kfree(fdev); @@ -1322,7 +1331,6 @@ static void fsldma_of_remove(struct platform_device *op) if (fdev->chan[i]) fsl_dma_chan_remove(fdev->chan[i]); } - irq_dispose_mapping(fdev->irq); iounmap(fdev->regs); kfree(fdev); diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c index 5fcd1befc92d..657629b100fa 100644 --- a/drivers/dma/idma64.c +++ b/drivers/dma/idma64.c @@ -25,11 +25,6 @@ /* ---------------------------------------------------------------------- */ -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - /* ---------------------------------------------------------------------- */ static void idma64_off(struct idma64 *idma64) @@ -507,11 +502,10 @@ static int idma64_alloc_chan_resources(struct dma_chan *chan) struct idma64_chan *idma64c = to_idma64_chan(chan); /* Create a pool of consistent memory blocks for hardware descriptors */ - idma64c->pool = dma_pool_create(dev_name(chan2dev(chan)), - chan->device->dev, + idma64c->pool = dma_pool_create(dma_chan_name(chan), chan->device->dev, sizeof(struct idma64_lli), 8, 0); if (!idma64c->pool) { - dev_err(chan2dev(chan), "No memory for descriptors\n"); + dev_err(dmaengine_chan_dev(chan), "No memory for descriptors\n"); return -ENOMEM; } diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c index b3765ba15803..a3192d49b6c8 100644 --- a/drivers/dma/img-mdc-dma.c +++ b/drivers/dma/img-mdc-dma.c @@ -738,7 +738,7 @@ static int mdc_alloc_chan_resources(struct dma_chan *chan) struct mdc_chan *mchan = to_mdc_chan(chan); struct device *dev = mdma2dev(mchan->mdma); - return pm_runtime_get_sync(dev); + return pm_runtime_resume_and_get(dev); } static void mdc_free_chan_resources(struct dma_chan *chan) diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index 737496391109..a57024c4b066 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -1170,15 +1170,6 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) ioat_chan->reg_base + IOAT_DCACTRL_OFFSET); } - err = dma_async_device_register(&ioat_dma->dma_dev); - if (err) - goto err_disable_interrupts; - - ioat_kobject_add(ioat_dma, &ioat_ktype); - - if (dca) - ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base); - /* disable relaxed ordering */ err = pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &val16); if (err) { @@ -1194,6 +1185,15 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) goto err_disable_interrupts; } + err = dma_async_device_register(&ioat_dma->dma_dev); + if (err) + goto err_disable_interrupts; + + ioat_kobject_add(ioat_dma, &ioat_ktype); + + if (dca) + ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base); + if (ioat_dma->cap & IOAT_CAP_DPS) writeb(ioat_pending_level + 1, ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET); diff --git a/drivers/dma/ioat/sysfs.c b/drivers/dma/ioat/sysfs.c index e796ddb5383f..976134df8108 100644 --- a/drivers/dma/ioat/sysfs.c +++ b/drivers/dma/ioat/sysfs.c @@ -24,12 +24,12 @@ static ssize_t cap_show(struct dma_chan *c, char *page) { struct dma_device *dma = c->device; - return sprintf(page, "copy%s%s%s%s%s\n", - dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "", - dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "", - dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "", - dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "", - dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : ""); + return sysfs_emit(page, "copy%s%s%s%s%s\n", + dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "", + dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "", + dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "", + dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "", + dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : ""); } static const struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); @@ -39,8 +39,8 @@ static ssize_t version_show(struct dma_chan *c, char *page) struct dma_device *dma = c->device; struct ioatdma_device *ioat_dma = to_ioatdma_device(dma); - return sprintf(page, "%d.%d\n", - ioat_dma->version >> 4, ioat_dma->version & 0xf); + return sysfs_emit(page, "%d.%d\n", + ioat_dma->version >> 4, ioat_dma->version & 0xf); } static const struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version); @@ -118,7 +118,7 @@ static ssize_t ring_size_show(struct dma_chan *c, char *page) { struct ioatdma_chan *ioat_chan = to_ioat_chan(c); - return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1); + return sysfs_emit(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1); } static const struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); @@ -127,7 +127,7 @@ static ssize_t ring_active_show(struct dma_chan *c, char *page) struct ioatdma_chan *ioat_chan = to_ioat_chan(c); /* ...taken outside the lock, no need to be precise */ - return sprintf(page, "%d\n", ioat_ring_active(ioat_chan)); + return sysfs_emit(page, "%d\n", ioat_ring_active(ioat_chan)); } static const struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); @@ -135,7 +135,7 @@ static ssize_t intr_coalesce_show(struct dma_chan *c, char *page) { struct ioatdma_chan *ioat_chan = to_ioat_chan(c); - return sprintf(page, "%d\n", ioat_chan->intr_coalesce); + return sysfs_emit(page, "%d\n", ioat_chan->intr_coalesce); } static ssize_t intr_coalesce_store(struct dma_chan *c, const char *page, diff --git a/drivers/dma/loongson/loongson1-apb-dma.c b/drivers/dma/loongson/loongson1-apb-dma.c index 89786cbd20ab..46b4bfef45e2 100644 --- a/drivers/dma/loongson/loongson1-apb-dma.c +++ b/drivers/dma/loongson/loongson1-apb-dma.c @@ -89,11 +89,6 @@ static irqreturn_t ls1x_dma_irq_handler(int irq, void *data); #define to_ls1x_dma_desc(d) \ container_of(d, struct ls1x_dma_desc, vd) -static inline struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - static inline int ls1x_dma_query(struct ls1x_dma_chan *chan, dma_addr_t *lli_phys) { @@ -107,7 +102,7 @@ static inline int ls1x_dma_query(struct ls1x_dma_chan *chan, ret = readl_poll_timeout_atomic(chan->reg_base + LS1X_DMA_CTRL, val, !(val & LS1X_DMA_ASK_VALID), 0, 3000); if (ret) - dev_err(chan2dev(dchan), "failed to query DMA\n"); + dev_err(dmaengine_chan_dev(dchan), "failed to query DMA\n"); return ret; } @@ -116,7 +111,7 @@ static inline int ls1x_dma_start(struct ls1x_dma_chan *chan, dma_addr_t *lli_phys) { struct dma_chan *dchan = &chan->vc.chan; - struct device *dev = chan2dev(dchan); + struct device *dev = dmaengine_chan_dev(dchan); int val, ret; val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK; @@ -143,7 +138,7 @@ static inline void ls1x_dma_stop(struct ls1x_dma_chan *chan) static void ls1x_dma_free_chan_resources(struct dma_chan *dchan) { struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); - struct device *dev = chan2dev(dchan); + struct device *dev = dmaengine_chan_dev(dchan); dma_free_coherent(dev, sizeof(struct ls1x_dma_lli), chan->curr_lli, chan->curr_lli->phys); @@ -156,7 +151,7 @@ static void ls1x_dma_free_chan_resources(struct dma_chan *dchan) static int ls1x_dma_alloc_chan_resources(struct dma_chan *dchan) { struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); - struct device *dev = chan2dev(dchan); + struct device *dev = dmaengine_chan_dev(dchan); dma_addr_t phys; int ret; @@ -219,7 +214,7 @@ static int ls1x_dma_prep_lli(struct dma_chan *dchan, struct ls1x_dma_desc *desc, { struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); struct ls1x_dma_lli *lli, *prev = NULL, *first = NULL; - struct device *dev = chan2dev(dchan); + struct device *dev = dmaengine_chan_dev(dchan); struct list_head *pos = NULL; struct scatterlist *sg; unsigned int dev_addr, cmd, i; @@ -299,7 +294,7 @@ ls1x_dma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, { struct ls1x_dma_desc *desc; - dev_dbg(chan2dev(dchan), "sg_len=%u flags=0x%lx dir=%s\n", + dev_dbg(dmaengine_chan_dev(dchan), "sg_len=%u flags=0x%lx dir=%s\n", sg_len, flags, dmaengine_get_direction_text(dir)); desc = ls1x_dma_alloc_desc(); @@ -325,7 +320,7 @@ ls1x_dma_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr, unsigned int i; int ret; - dev_dbg(chan2dev(dchan), + dev_dbg(dmaengine_chan_dev(dchan), "buf_len=%zu period_len=%zu flags=0x%lx dir=%s\n", buf_len, period_len, flags, dmaengine_get_direction_text(dir)); @@ -450,7 +445,7 @@ static enum dma_status ls1x_dma_tx_status(struct dma_chan *dchan, if (lli->hw[LS1X_DMADESC_NEXT] == next_phys) break; - dev_dbg(chan2dev(dchan), "current lli_phys=%pad", + dev_dbg(dmaengine_chan_dev(dchan), "current lli_phys=%pad", &lli->phys); /* count the residues */ @@ -489,7 +484,7 @@ static irqreturn_t ls1x_dma_irq_handler(int irq, void *data) { struct ls1x_dma_chan *chan = data; struct dma_chan *dchan = &chan->vc.chan; - struct device *dev = chan2dev(dchan); + struct device *dev = dmaengine_chan_dev(dchan); struct virt_dma_desc *vd; scoped_guard(spinlock, &chan->vc.lock) { diff --git a/drivers/dma/loongson/loongson2-apb-cmc-dma.c b/drivers/dma/loongson/loongson2-apb-cmc-dma.c index 1c9a542edc85..2037120b23ca 100644 --- a/drivers/dma/loongson/loongson2-apb-cmc-dma.c +++ b/drivers/dma/loongson/loongson2-apb-cmc-dma.c @@ -90,7 +90,7 @@ struct loongson2_cmc_dma_chan { struct dma_slave_config dma_sconfig; struct loongson2_cmc_dma_desc *desc; u32 id; - u32 irq; + int irq; u32 next_sg; struct loongson2_cmc_dma_chan_reg chan_reg; }; @@ -134,11 +134,6 @@ static struct loongson2_cmc_dma_desc *to_lmdma_desc(struct virt_dma_desc *vdesc) return container_of(vdesc, struct loongson2_cmc_dma_desc, vdesc); } -static struct device *chan2dev(struct loongson2_cmc_dma_chan *lchan) -{ - return &lchan->vchan.chan.dev->device; -} - static u32 loongson2_cmc_dma_read(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id) { return readl(lddev->base + (reg + lddev->chan_reg_offset * id)); @@ -302,7 +297,7 @@ static irqreturn_t loongson2_cmc_dma_chan_irq(int irq, void *devid) { struct loongson2_cmc_dma_chan *lchan = devid; struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); - struct device *dev = chan2dev(lchan); + struct device *dev = vchan_chan_dev(&lchan->vchan); u32 ists, status, ccr; scoped_guard(spinlock, &lchan->vchan.lock) { @@ -337,7 +332,7 @@ static void loongson2_cmc_dma_issue_pending(struct dma_chan *chan) guard(spinlock_irqsave)(&lchan->vchan.lock); if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) { - dev_dbg(chan2dev(lchan), "vchan %pK: issued\n", &lchan->vchan); + dev_dbg(vchan_chan_dev(&lchan->vchan), "vchan %pK: issued\n", &lchan->vchan); loongson2_cmc_dma_start_transfer(lchan); } } @@ -347,7 +342,7 @@ static int loongson2_cmc_dma_set_xfer_param(struct loongson2_cmc_dma_chan *lchan enum dma_slave_buswidth *buswidth, u32 buf_len) { struct dma_slave_config sconfig = lchan->dma_sconfig; - struct device *dev = chan2dev(lchan); + struct device *dev = vchan_chan_dev(&lchan->vchan); int dev_width; u32 ccr; @@ -404,12 +399,14 @@ loongson2_cmc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, for_each_sg(sgl, sg, sg_len, i) { ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, sg_dma_len(sg)); - if (ret) + if (ret) { + kfree(desc); return ERR_PTR(ret); + } num_items = DIV_ROUND_UP(sg_dma_len(sg), buswidth); if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) { - dev_err(chan2dev(lchan), "Number of items not supported\n"); + dev_err(vchan_chan_dev(&lchan->vchan), "Number of items not supported\n"); kfree(desc); return ERR_PTR(-EINVAL); } @@ -447,7 +444,7 @@ loongson2_cmc_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, si num_items = DIV_ROUND_UP(period_len, buswidth); if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) { - dev_err(chan2dev(lchan), "Number of items not supported\n"); + dev_err(vchan_chan_dev(&lchan->vchan), "Number of items not supported\n"); return ERR_PTR(-EINVAL); } @@ -678,7 +675,7 @@ static int loongson2_cmc_dma_probe(struct platform_device *pdev) return lchan->irq; ret = devm_request_irq(dev, lchan->irq, loongson2_cmc_dma_chan_irq, IRQF_SHARED, - dev_name(chan2dev(lchan)), lchan); + vchan_chan_name(&lchan->vchan), lchan); if (ret) return ret; } @@ -709,8 +706,8 @@ static const struct of_device_id loongson2_cmc_dma_of_match[] = { MODULE_DEVICE_TABLE(of, loongson2_cmc_dma_of_match); static const struct acpi_device_id loongson2_cmc_dma_acpi_match[] = { - { "LOON0014", .driver_data = (kernel_ulong_t)&ls2k3000_cmc_dma_config }, - { /* sentinel */ } + { .id = "LOON0014", .driver_data = (kernel_ulong_t)&ls2k3000_cmc_dma_config }, + { } }; MODULE_DEVICE_TABLE(acpi, loongson2_cmc_dma_acpi_match); diff --git a/drivers/dma/loongson/loongson2-apb-dma.c b/drivers/dma/loongson/loongson2-apb-dma.c index aceb069e71fc..16a0c1c60d59 100644 --- a/drivers/dma/loongson/loongson2-apb-dma.c +++ b/drivers/dma/loongson/loongson2-apb-dma.c @@ -161,11 +161,6 @@ static inline struct ls2x_dma_priv *to_ldma_priv(struct dma_device *ddev) return container_of(ddev, struct ls2x_dma_priv, ddev); } -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - static void ls2x_dma_desc_free(struct virt_dma_desc *vdesc) { struct ls2x_dma_chan *lchan = to_ldma_chan(vdesc->tx.chan); @@ -282,11 +277,11 @@ static int ls2x_dma_alloc_chan_resources(struct dma_chan *chan) struct ls2x_dma_chan *lchan = to_ldma_chan(chan); /* Create a pool of consistent memory blocks for hardware descriptors */ - lchan->pool = dma_pool_create(dev_name(chan2dev(chan)), + lchan->pool = dma_pool_create(dma_chan_name(chan), chan->device->dev, PAGE_SIZE, __alignof__(struct ls2x_dma_hw_desc), 0); if (!lchan->pool) { - dev_err(chan2dev(chan), "No memory for descriptors\n"); + dev_err(dmaengine_chan_dev(chan), "No memory for descriptors\n"); return -ENOMEM; } diff --git a/drivers/dma/mediatek/mtk-hsdma.c b/drivers/dma/mediatek/mtk-hsdma.c index a43412ff5edd..27cb370106c7 100644 --- a/drivers/dma/mediatek/mtk-hsdma.c +++ b/drivers/dma/mediatek/mtk-hsdma.c @@ -848,17 +848,27 @@ static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma) { int err; - pm_runtime_enable(hsdma2dev(hsdma)); - pm_runtime_get_sync(hsdma2dev(hsdma)); + err = devm_pm_runtime_enable(hsdma2dev(hsdma)); + if (err) + return err; + + err = pm_runtime_resume_and_get(hsdma2dev(hsdma)); + if (err < 0) + return err; err = clk_prepare_enable(hsdma->clk); if (err) - return err; + goto err_put_pm; mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0); mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT); return 0; + +err_put_pm: + pm_runtime_put_sync(hsdma2dev(hsdma)); + + return err; } static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma) @@ -868,7 +878,6 @@ static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma) clk_disable_unprepare(hsdma->clk); pm_runtime_put_sync(hsdma2dev(hsdma)); - pm_runtime_disable(hsdma2dev(hsdma)); return 0; } @@ -983,7 +992,9 @@ static int mtk_hsdma_probe(struct platform_device *pdev) goto err_unregister; } - mtk_hsdma_hw_init(hsdma); + err = mtk_hsdma_hw_init(hsdma); + if (err) + goto err_free; err = devm_request_irq(&pdev->dev, hsdma->irq, mtk_hsdma_irq, 0, @@ -991,7 +1002,7 @@ static int mtk_hsdma_probe(struct platform_device *pdev) if (err) { dev_err(&pdev->dev, "request_irq failed with err %d\n", err); - goto err_free; + goto err_deinit; } platform_set_drvdata(pdev, hsdma); @@ -1000,8 +1011,9 @@ static int mtk_hsdma_probe(struct platform_device *pdev) return 0; -err_free: +err_deinit: mtk_hsdma_hw_deinit(hsdma); +err_free: of_dma_controller_free(pdev->dev.of_node); err_unregister: dma_async_device_unregister(dd); diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index ed520737882b..8abd24e2ffc7 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -564,8 +564,7 @@ static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan) if (chan->desc_pool) return 1; - chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device), - chan->dev, + chan->desc_pool = dma_pool_create(dma_chan_name(dchan), chan->dev, sizeof(struct mmp_pdma_desc_sw), __alignof__(struct mmp_pdma_desc_sw), 0); diff --git a/drivers/dma/moxart-dma.c b/drivers/dma/moxart-dma.c index 442f5aa16031..28be93eed4cc 100644 --- a/drivers/dma/moxart-dma.c +++ b/drivers/dma/moxart-dma.c @@ -154,11 +154,6 @@ static const unsigned int es_bytes[] = { [MOXART_DMA_DATA_TYPE_S32] = 4, }; -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - static inline struct moxart_chan *to_moxart_dma_chan(struct dma_chan *c) { return container_of(c, struct moxart_chan, vc.chan); @@ -182,7 +177,7 @@ static int moxart_terminate_all(struct dma_chan *chan) LIST_HEAD(head); u32 ctrl; - dev_dbg(chan2dev(chan), "%s: ch=%p\n", __func__, ch); + dev_dbg(dmaengine_chan_dev(chan), "%s: ch=%p\n", __func__, ch); spin_lock_irqsave(&ch->vc.lock, flags); @@ -272,7 +267,7 @@ static struct dma_async_tx_descriptor *moxart_prep_slave_sg( unsigned int i; if (!is_slave_direction(dir)) { - dev_err(chan2dev(chan), "%s: invalid DMA direction\n", + dev_err(dmaengine_chan_dev(chan), "%s: invalid DMA direction\n", __func__); return NULL; } @@ -296,7 +291,7 @@ static struct dma_async_tx_descriptor *moxart_prep_slave_sg( es = MOXART_DMA_DATA_TYPE_S32; break; default: - dev_err(chan2dev(chan), "%s: unsupported data width (%u)\n", + dev_err(dmaengine_chan_dev(chan), "%s: unsupported data width (%u)\n", __func__, dev_width); return NULL; } @@ -341,7 +336,7 @@ static int moxart_alloc_chan_resources(struct dma_chan *chan) { struct moxart_chan *ch = to_moxart_dma_chan(chan); - dev_dbg(chan2dev(chan), "%s: allocating channel #%u\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: allocating channel #%u\n", __func__, ch->ch_num); ch->allocated = 1; @@ -354,7 +349,7 @@ static void moxart_free_chan_resources(struct dma_chan *chan) vchan_free_chan_resources(&ch->vc); - dev_dbg(chan2dev(chan), "%s: freeing channel #%u\n", + dev_dbg(dmaengine_chan_dev(chan), "%s: freeing channel #%u\n", __func__, ch->ch_num); ch->allocated = 0; } @@ -379,7 +374,7 @@ static void moxart_set_transfer_params(struct moxart_chan *ch, unsigned int len) */ writel(d->dma_cycles, ch->base + REG_OFF_CYCLES); - dev_dbg(chan2dev(&ch->vc.chan), "%s: set %u DMA cycles (len=%u)\n", + dev_dbg(vchan_chan_dev(&ch->vc), "%s: set %u DMA cycles (len=%u)\n", __func__, d->dma_cycles, len); } @@ -460,7 +455,7 @@ static size_t moxart_dma_desc_size_in_flight(struct moxart_chan *ch) completed_cycles = (ch->desc->dma_cycles - cycles); size -= completed_cycles << es_bytes[ch->desc->es]; - dev_dbg(chan2dev(&ch->vc.chan), "%s: size=%zu\n", __func__, size); + dev_dbg(vchan_chan_dev(&ch->vc), "%s: size=%zu\n", __func__, size); return size; } @@ -517,7 +512,7 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid) unsigned int i; u32 ctrl; - dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__); + dev_dbg(vchan_chan_dev(&ch->vc), "%s\n", __func__); for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) { if (!ch->allocated) @@ -525,7 +520,7 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid) ctrl = readl(ch->base + REG_OFF_CTRL); - dev_dbg(chan2dev(&ch->vc.chan), "%s: ch=%p ch->base=%p ctrl=%x\n", + dev_dbg(vchan_chan_dev(&ch->vc), "%s: ch=%p ch->base=%p ctrl=%x\n", __func__, ch, ch->base, ctrl); if (ctrl & APB_DMA_FIN_INT_STS) { diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 25ed61f1b089..c4e0dce3ca64 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -583,8 +583,13 @@ mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, while (src_cnt--) { /* Check if a new window needs to get added for 'src' */ ret = mv_xor_add_io_win(mv_chan, src[src_cnt]); - if (ret) + if (ret) { + spin_lock_bh(&mv_chan->lock); + list_move_tail(&sw_desc->node, + &mv_chan->free_slots); + spin_unlock_bh(&mv_chan->lock); return NULL; + } mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]); } } @@ -1162,6 +1167,7 @@ mv_xor_channel_add(struct mv_xor_device *xordev, err_free_irq: free_irq(mv_chan->irq, mv_chan); + tasklet_kill(&mv_chan->irq_tasklet); err_free_dma: dma_free_wc(&pdev->dev, MV_XOR_POOL_SIZE, mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c index 05d7321629cc..744cd14d7872 100644 --- a/drivers/dma/nbpfaxi.c +++ b/drivers/dma/nbpfaxi.c @@ -1202,7 +1202,7 @@ static irqreturn_t nbpf_chan_irq(int irq, void *dev) nbpf_status_ack(chan); - dev_dbg(&chan->dma_chan.dev->device, "%s()\n", __func__); + dev_dbg(dmaengine_chan_dev(&chan->dma_chan), "%s()\n", __func__); spin_lock(&chan->lock); desc = chan->running; diff --git a/drivers/dma/owl-dma.c b/drivers/dma/owl-dma.c index 7c80572fc71d..9dc88d804a2e 100644 --- a/drivers/dma/owl-dma.c +++ b/drivers/dma/owl-dma.c @@ -292,11 +292,6 @@ static inline struct owl_dma *to_owl_dma(struct dma_device *dd) return container_of(dd, struct owl_dma, dma); } -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan) { return container_of(chan, struct owl_dma_vchan, vc.chan); @@ -573,7 +568,7 @@ static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan) spin_unlock_irqrestore(&od->lock, flags); - dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id); + dev_dbg(vchan_chan_dev(&vchan->vc), "starting pchan %d\n", pchan->id); /* Start DMA transfer for this pchan */ pchan_writel(pchan, OWL_DMAX_START, 0x1); @@ -757,7 +752,7 @@ static int owl_dma_resume(struct dma_chan *chan) if (!vchan->pchan && !vchan->txd) return 0; - dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc); + dev_dbg(dmaengine_chan_dev(chan), "vchan %p: resume\n", &vchan->vc); spin_lock_irqsave(&vchan->vc.lock, flags); @@ -888,7 +883,7 @@ static struct dma_async_tx_descriptor for (offset = 0; offset < len; offset += bytes) { lli = owl_dma_alloc_lli(od); if (!lli) { - dev_warn(chan2dev(chan), "failed to allocate lli\n"); + dev_warn(dmaengine_chan_dev(chan), "failed to allocate lli\n"); goto err_txd_free; } @@ -898,7 +893,7 @@ static struct dma_async_tx_descriptor bytes, DMA_MEM_TO_MEM, &vchan->cfg, txd->cyclic); if (ret) { - dev_warn(chan2dev(chan), "failed to config lli\n"); + dev_warn(dmaengine_chan_dev(chan), "failed to config lli\n"); goto err_txd_free; } @@ -947,7 +942,7 @@ static struct dma_async_tx_descriptor lli = owl_dma_alloc_lli(od); if (!lli) { - dev_err(chan2dev(chan), "failed to allocate lli"); + dev_err(dmaengine_chan_dev(chan), "failed to allocate lli"); goto err_txd_free; } @@ -962,7 +957,7 @@ static struct dma_async_tx_descriptor ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig, txd->cyclic); if (ret) { - dev_warn(chan2dev(chan), "failed to config lli"); + dev_warn(dmaengine_chan_dev(chan), "failed to config lli"); goto err_txd_free; } @@ -1003,7 +998,7 @@ static struct dma_async_tx_descriptor for (i = 0; i < periods; i++) { lli = owl_dma_alloc_lli(od); if (!lli) { - dev_warn(chan2dev(chan), "failed to allocate lli"); + dev_warn(dmaengine_chan_dev(chan), "failed to allocate lli"); goto err_txd_free; } @@ -1018,7 +1013,7 @@ static struct dma_async_tx_descriptor ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len, dir, sconfig, txd->cyclic); if (ret) { - dev_warn(chan2dev(chan), "failed to config lli"); + dev_warn(dmaengine_chan_dev(chan), "failed to config lli"); goto err_txd_free; } diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index bf805f1024f6..88baf7ffec37 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -150,11 +150,6 @@ static inline struct pch_dma *to_pd(struct dma_device *ddev) return container_of(ddev, struct pch_dma, dma); } -static inline struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - static inline struct pch_dma_desc *pdc_first_active(struct pch_dma_chan *pd_chan) { @@ -189,7 +184,7 @@ static void pdc_enable_irq(struct dma_chan *chan, int enable) dma_writel(pd, CTL2, val); - dev_dbg(chan2dev(chan), "pdc_enable_irq: chan %d -> %x\n", + dev_dbg(dmaengine_chan_dev(chan), "pdc_enable_irq: chan %d -> %x\n", chan->chan_id, val); } @@ -237,7 +232,7 @@ static void pdc_set_dir(struct dma_chan *chan) dma_writel(pd, CTL3, val); } - dev_dbg(chan2dev(chan), "pdc_set_dir: chan %d -> %x\n", + dev_dbg(dmaengine_chan_dev(chan), "pdc_set_dir: chan %d -> %x\n", chan->chan_id, val); } @@ -271,7 +266,7 @@ static void pdc_set_mode(struct dma_chan *chan, u32 mode) dma_writel(pd, CTL3, val); } - dev_dbg(chan2dev(chan), "pdc_set_mode: chan %d -> %x\n", + dev_dbg(dmaengine_chan_dev(chan), "pdc_set_mode: chan %d -> %x\n", chan->chan_id, val); } @@ -314,18 +309,18 @@ static bool pdc_is_idle(struct pch_dma_chan *pd_chan) static void pdc_dostart(struct pch_dma_chan *pd_chan, struct pch_dma_desc* desc) { if (!pdc_is_idle(pd_chan)) { - dev_err(chan2dev(&pd_chan->chan), + dev_err(dmaengine_chan_dev(&pd_chan->chan), "BUG: Attempt to start non-idle channel\n"); return; } - dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> dev_addr: %x\n", + dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "chan %d -> dev_addr: %x\n", pd_chan->chan.chan_id, desc->regs.dev_addr); - dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> mem_addr: %x\n", + dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "chan %d -> mem_addr: %x\n", pd_chan->chan.chan_id, desc->regs.mem_addr); - dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> size: %x\n", + dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "chan %d -> size: %x\n", pd_chan->chan.chan_id, desc->regs.size); - dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> next: %x\n", + dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "chan %d -> next: %x\n", pd_chan->chan.chan_id, desc->regs.next); if (list_empty(&desc->tx_list)) { @@ -382,8 +377,8 @@ static void pdc_handle_error(struct pch_dma_chan *pd_chan) if (!list_empty(&pd_chan->active_list)) pdc_dostart(pd_chan, pdc_first_active(pd_chan)); - dev_crit(chan2dev(&pd_chan->chan), "Bad descriptor submitted\n"); - dev_crit(chan2dev(&pd_chan->chan), "descriptor cookie: %d\n", + dev_crit(dmaengine_chan_dev(&pd_chan->chan), "Bad descriptor submitted\n"); + dev_crit(dmaengine_chan_dev(&pd_chan->chan), "descriptor cookie: %d\n", bad_desc->txd.cookie); pdc_chain_complete(pd_chan, bad_desc); @@ -450,10 +445,10 @@ static struct pch_dma_desc *pdc_desc_get(struct pch_dma_chan *pd_chan) ret = desc; break; } - dev_dbg(chan2dev(&pd_chan->chan), "desc %p not ACKed\n", desc); + dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "desc %p not ACKed\n", desc); } spin_unlock(&pd_chan->lock); - dev_dbg(chan2dev(&pd_chan->chan), "scanned %d descriptors\n", i); + dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "scanned %d descriptors\n", i); if (!ret) { ret = pdc_alloc_desc(&pd_chan->chan, GFP_ATOMIC); @@ -462,7 +457,7 @@ static struct pch_dma_desc *pdc_desc_get(struct pch_dma_chan *pd_chan) pd_chan->descs_allocated++; spin_unlock(&pd_chan->lock); } else { - dev_err(chan2dev(&pd_chan->chan), + dev_err(dmaengine_chan_dev(&pd_chan->chan), "failed to alloc desc\n"); } } @@ -489,7 +484,7 @@ static int pd_alloc_chan_resources(struct dma_chan *chan) int i; if (!pdc_is_idle(pd_chan)) { - dev_dbg(chan2dev(chan), "DMA channel not idle ?\n"); + dev_dbg(dmaengine_chan_dev(chan), "DMA channel not idle ?\n"); return -EIO; } @@ -500,7 +495,7 @@ static int pd_alloc_chan_resources(struct dma_chan *chan) desc = pdc_alloc_desc(chan, GFP_KERNEL); if (!desc) { - dev_warn(chan2dev(chan), + dev_warn(dmaengine_chan_dev(chan), "Only allocated %d initial descriptors\n", i); break; } @@ -573,7 +568,7 @@ static struct dma_async_tx_descriptor *pd_prep_slave_sg(struct dma_chan *chan, int i; if (unlikely(!sg_len)) { - dev_info(chan2dev(chan), "prep_slave_sg: length is zero!\n"); + dev_info(dmaengine_chan_dev(chan), "prep_slave_sg: length is zero!\n"); return NULL; } @@ -639,7 +634,7 @@ static struct dma_async_tx_descriptor *pd_prep_slave_sg(struct dma_chan *chan, return &first->txd; err_desc_get: - dev_err(chan2dev(chan), "failed to get desc or wrong parameters\n"); + dev_err(dmaengine_chan_dev(chan), "failed to get desc or wrong parameters\n"); pdc_desc_put(pd_chan, first); return NULL; } @@ -671,7 +666,7 @@ static void pdc_tasklet(struct tasklet_struct *t) unsigned long flags; if (!pdc_is_idle(pd_chan)) { - dev_err(chan2dev(&pd_chan->chan), + dev_err(dmaengine_chan_dev(&pd_chan->chan), "BUG: handle non-idle channel in tasklet\n"); return; } diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 6214d9000db8..c2d0b473b8f4 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -2953,8 +2953,8 @@ DEFINE_SHOW_ATTRIBUTE(pl330_debugfs); static inline void init_pl330_debugfs(struct pl330_dmac *pl330) { - pl330->dbgfs = debugfs_create_file(dev_name(pl330->ddma.dev), - S_IFREG | 0444, NULL, pl330, + pl330->dbgfs = debugfs_create_file("stats", 0444, + pl330->ddma.dbg_dev_root, pl330, &pl330_debugfs_fops); } diff --git a/drivers/dma/plx_dma.c b/drivers/dma/plx_dma.c index 84941a918b01..409898e92c32 100644 --- a/drivers/dma/plx_dma.c +++ b/drivers/dma/plx_dma.c @@ -504,17 +504,17 @@ static int plx_dma_create(struct pci_dev *pdev) if (!plxdev) return -ENOMEM; - rc = request_irq(pci_irq_vector(pdev, 0), plx_dma_isr, 0, - KBUILD_MODNAME, plxdev); - if (rc) - goto free_plx; - spin_lock_init(&plxdev->ring_lock); tasklet_setup(&plxdev->desc_task, plx_dma_desc_task); RCU_INIT_POINTER(plxdev->pdev, pdev); plxdev->bar = pcim_iomap_table(pdev)[0]; + rc = request_irq(pci_irq_vector(pdev, 0), plx_dma_isr, 0, + KBUILD_MODNAME, plxdev); + if (rc) + goto free_plx; + dma = &plxdev->dma_dev; INIT_LIST_HEAD(&dma->channels); dma_cap_set(DMA_MEMCPY, dma->cap_mask); diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index 279a431ccae3..bcc54d584e2b 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -799,7 +799,7 @@ static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) return slot_cnt; - printk(KERN_ERR "%s: len %d > max %d !!\n", + printk(KERN_ERR "%s: len %zu > max %d !!\n", __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); BUG(); return slot_cnt; @@ -1989,7 +1989,7 @@ static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( spin_lock_bh(&ppc440spe_chan->lock); dev_dbg(ppc440spe_chan->device->common.dev, - "ppc440spe adma%d: %s len: %u int_en %d\n", + "ppc440spe adma%d: %s len: %zu int_en %d\n", ppc440spe_chan->device->id, __func__, len, flags & DMA_PREP_INTERRUPT ? 1 : 0); slot_cnt = slots_per_op = 1; @@ -2030,7 +2030,7 @@ static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); dev_dbg(ppc440spe_chan->device->common.dev, - "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", + "ppc440spe adma%d: %s src_cnt: %d len: %zu int_en: %d\n", ppc440spe_chan->device->id, __func__, src_cnt, len, flags & DMA_PREP_INTERRUPT ? 1 : 0); @@ -2279,7 +2279,7 @@ static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( unsigned long op = 0; unsigned char mult = 1; - pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", + pr_debug("%s: dst_cnt %d, src_cnt %d, len %zu\n", __func__, dst_cnt, src_cnt, len); /* select operations WXOR/RXOR depending on the * source addresses of operators and the number @@ -2553,7 +2553,7 @@ static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( BUG_ON(!dst_cnt); dev_dbg(ppc440spe_chan->device->common.dev, - "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", + "ppc440spe adma%d: %s src_cnt: %d len: %zu int_en: %d\n", ppc440spe_chan->device->id, __func__, src_cnt, len, flags & DMA_PREP_INTERRUPT ? 1 : 0); @@ -4077,7 +4077,7 @@ static int ppc440spe_adma_probe(struct platform_device *ofdev) adev->pool_size, &adev->dma_desc_pool, GFP_KERNEL); if (adev->dma_desc_pool_virt == NULL) { - dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " + dev_err(&ofdev->dev, "failed to allocate %zu bytes of coherent " "memory for hardware descriptors\n", adev->pool_size); initcode = PPC_ADMA_INIT_COHERENT; diff --git a/drivers/dma/ppc4xx/xor.h b/drivers/dma/ppc4xx/xor.h index da1230df2817..0931e18cd0d3 100644 --- a/drivers/dma/ppc4xx/xor.h +++ b/drivers/dma/ppc4xx/xor.h @@ -12,7 +12,7 @@ #include -/* Number of XOR engines available on the contoller */ +/* Number of XOR engines available on the controller */ #define XOR_ENGINES_NUM 1 /* Number of operands supported in the h/w */ diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index fc43124fefa8..4f0d1d531f01 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -149,7 +149,7 @@ struct pxad_device { ({ \ u32 _v; \ _v = readl_relaxed((phy)->base + _reg((phy)->idx)); \ - dev_vdbg(&phy->vchan->vc.chan.dev->device, \ + dev_vdbg(vchan_chan_dev(&phy->vchan->vc), \ "%s(): readl(%s): 0x%08x\n", __func__, #_reg, \ _v); \ _v; \ @@ -157,14 +157,14 @@ struct pxad_device { #define phy_writel(phy, val, _reg) \ do { \ writel((val), (phy)->base + _reg((phy)->idx)); \ - dev_vdbg(&phy->vchan->vc.chan.dev->device, \ + dev_vdbg(vchan_chan_dev(&phy->vchan->vc), \ "%s(): writel(0x%08x, %s)\n", \ __func__, (u32)(val), #_reg); \ } while (0) #define phy_writel_relaxed(phy, val, _reg) \ do { \ writel_relaxed((val), (phy)->base + _reg((phy)->idx)); \ - dev_vdbg(&phy->vchan->vc.chan.dev->device, \ + dev_vdbg(vchan_chan_dev(&phy->vchan->vc), \ "%s(): writel_relaxed(0x%08x, %s)\n", \ __func__, (u32)(val), #_reg); \ } while (0) @@ -396,7 +396,7 @@ static struct pxad_phy *lookup_phy(struct pxad_chan *pchan) out_unlock: spin_unlock_irqrestore(&pdev->phy_lock, flags); - dev_dbg(&pchan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&pchan->vc), "%s(): phy=%p(%d)\n", __func__, found, found ? found->idx : -1); @@ -409,7 +409,7 @@ static void pxad_free_phy(struct pxad_chan *chan) unsigned long flags; u32 reg; - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): freeing\n", __func__); if (!chan->phy) return; @@ -454,7 +454,7 @@ static void phy_enable(struct pxad_phy *phy, bool misaligned) if (!phy->vchan) return; - dev_dbg(&phy->vchan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&phy->vchan->vc), "%s(); phy=%p(%d) misaligned=%d\n", __func__, phy, phy->idx, misaligned); @@ -483,7 +483,7 @@ static void phy_disable(struct pxad_phy *phy) return; dcsr = phy_readl_relaxed(phy, DCSR); - dev_dbg(&phy->vchan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&phy->vchan->vc), "%s(): phy=%p(%d)\n", __func__, phy, phy->idx); phy_writel(phy, dcsr & ~PXA_DCSR_RUN & ~PXA_DCSR_STOPIRQEN, DCSR); } @@ -491,12 +491,12 @@ static void phy_disable(struct pxad_phy *phy) static void pxad_launch_chan(struct pxad_chan *chan, struct pxad_desc_sw *desc) { - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): desc=%p\n", __func__, desc); if (!chan->phy) { chan->phy = lookup_phy(chan); if (!chan->phy) { - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): no free dma channel\n", __func__); return; } @@ -592,7 +592,7 @@ static unsigned int clear_chan_irq(struct pxad_phy *phy) dcsr = phy_readl_relaxed(phy, DCSR); phy_writel(phy, dcsr, DCSR); if ((dcsr & PXA_DCSR_BUSERR) && (phy->vchan)) - dev_warn(&phy->vchan->vc.chan.dev->device, + dev_warn(vchan_chan_dev(&phy->vchan->vc), "%s(chan=%p): PXA_DCSR_BUSERR\n", __func__, &phy->vchan); @@ -617,7 +617,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id) spin_lock(&chan->vc.lock); list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) { vd_completed = is_desc_completed(vd); - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): checking txd %p[%x]: completed=%d dcsr=0x%x\n", __func__, vd, vd->tx.cookie, vd_completed, dcsr); @@ -640,7 +640,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id) } if (!chan->bus_error && dcsr & PXA_DCSR_STOPSTATE) { - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): channel stopped, submitted_empty=%d issued_empty=%d", __func__, list_empty(&chan->vc.desc_submitted), @@ -694,7 +694,7 @@ static int pxad_alloc_chan_resources(struct dma_chan *dchan) __alignof__(struct pxad_desc_hw), 0); if (!chan->desc_pool) { - dev_err(&chan->vc.chan.dev->device, + dev_err(vchan_chan_dev(&chan->vc), "%s(): unable to allocate descriptor pool\n", __func__); return -ENOMEM; @@ -750,7 +750,7 @@ pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc) for (i = 0; i < nb_hw_desc; i++) { desc = dma_pool_alloc(sw_desc->desc_pool, GFP_NOWAIT, &dma); if (!desc) { - dev_err(&chan->vc.chan.dev->device, + dev_err(vchan_chan_dev(&chan->vc), "%s(): Couldn't allocate the %dth hw_desc from dma_pool %p\n", __func__, i, sw_desc->desc_pool); sw_desc->nb_desc = i; @@ -787,7 +787,7 @@ static dma_cookie_t pxad_tx_submit(struct dma_async_tx_descriptor *tx) if (list_empty(&vc->desc_submitted) && pxad_try_hotchain(vc, vd)) { list_move_tail(&vd->node, &vc->desc_issued); - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): txd %p[%x]: submitted (hot linked)\n", __func__, vd, cookie); goto out; @@ -810,7 +810,7 @@ static dma_cookie_t pxad_tx_submit(struct dma_async_tx_descriptor *tx) else vd_chained = NULL; } - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): txd %p[%x]: submitted (%s linked)\n", __func__, vd, cookie, vd_chained ? "cold" : "not"); list_move_tail(&vd->node, &vc->desc_submitted); @@ -833,7 +833,7 @@ static void pxad_issue_pending(struct dma_chan *dchan) vd_first = list_first_entry(&chan->vc.desc_submitted, struct virt_dma_desc, node); - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): txd %p[%x]", __func__, vd_first, vd_first->tx.cookie); vchan_issue_pending(&chan->vc); @@ -853,7 +853,7 @@ pxad_tx_prep(struct virt_dma_chan *vc, struct virt_dma_desc *vd, INIT_LIST_HEAD(&vd->node); tx = vchan_tx_prep(vc, vd, tx_flags); tx->tx_submit = pxad_tx_submit; - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): vc=%p txd=%p[%x] flags=0x%lx\n", __func__, vc, vd, vd->tx.cookie, tx_flags); @@ -892,7 +892,7 @@ static void pxad_get_config(struct pxad_chan *chan, *dcmd |= PXA_DCMD_BURST32 | PXA_DCMD_INCTRGADDR | PXA_DCMD_INCSRCADDR; - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): dev_addr=0x%x maxburst=%d width=%d dir=%d\n", __func__, dev_addr, maxburst, width, dir); @@ -926,7 +926,7 @@ pxad_prep_memcpy(struct dma_chan *dchan, if (!dchan || !len) return NULL; - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): dma_dst=0x%lx dma_src=0x%lx len=%zu flags=%lx\n", __func__, (unsigned long)dma_dst, (unsigned long)dma_src, len, flags); @@ -975,7 +975,7 @@ pxad_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, return NULL; pxad_get_config(chan, dir, &dcmd, &dsadr, &dtadr); - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): dir=%d flags=%lx\n", __func__, dir, flags); nb_desc = sg_nents_for_dma(sgl, sg_len, PDMA_MAX_DESC_BYTES); @@ -1022,7 +1022,7 @@ pxad_prep_dma_cyclic(struct dma_chan *dchan, if (!dchan || !len || !period_len) return NULL; if ((dir != DMA_DEV_TO_MEM) && (dir != DMA_MEM_TO_DEV)) { - dev_err(&chan->vc.chan.dev->device, + dev_err(vchan_chan_dev(&chan->vc), "Unsupported direction for cyclic DMA\n"); return NULL; } @@ -1033,7 +1033,7 @@ pxad_prep_dma_cyclic(struct dma_chan *dchan, pxad_get_config(chan, dir, &dcmd, &dsadr, &dtadr); dcmd |= PXA_DCMD_ENDIRQEN | (PXA_DCMD_LENGTH & period_len); - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): buf_addr=0x%lx len=%zu period=%zu dir=%d flags=%lx\n", __func__, (unsigned long)buf_addr, len, period_len, dir, flags); @@ -1081,14 +1081,14 @@ static int pxad_terminate_all(struct dma_chan *dchan) struct pxad_phy *phy; LIST_HEAD(head); - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): vchan %p: terminate all\n", __func__, &chan->vc); spin_lock_irqsave(&chan->vc.lock, flags); vchan_get_all_descriptors(&chan->vc, &head); list_for_each_entry(vd, &head, node) { - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): cancelling txd %p[%x] (completed=%d)", __func__, vd, vd->tx.cookie, is_desc_completed(vd)); } @@ -1178,7 +1178,7 @@ static unsigned int pxad_residue(struct pxad_chan *chan, out: spin_unlock_irqrestore(&chan->vc.lock, flags); - dev_dbg(&chan->vc.chan.dev->device, + dev_dbg(vchan_chan_dev(&chan->vc), "%s(): txd %p[%x] sw_desc=%p: %d\n", __func__, vd, cookie, sw_desc, residue); return residue; diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 05a3b1f9e0c2..f67a93a37d48 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -1332,8 +1332,8 @@ static int bam_dma_probe(struct platform_device *pdev) for (i = 0; i < bdev->num_channels; i++) bam_channel_init(bdev, &bdev->channels[i], i); - ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq, - IRQF_TRIGGER_HIGH, "bam_dma", bdev); + ret = request_irq(bdev->irq, bam_dma_irq, IRQF_TRIGGER_NONE, + "bam_dma", bdev); if (ret) goto err_bam_channel_exit; @@ -1366,7 +1366,7 @@ static int bam_dma_probe(struct platform_device *pdev) ret = dma_async_device_register(&bdev->common); if (ret) { dev_err(bdev->dev, "failed to register dma async device\n"); - goto err_bam_channel_exit; + goto err_free_irq; } ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate, @@ -1385,6 +1385,8 @@ static int bam_dma_probe(struct platform_device *pdev) err_unregister_dma: dma_async_device_unregister(&bdev->common); +err_free_irq: + free_irq(bdev->irq, bdev); err_bam_channel_exit: for (i = 0; i < bdev->num_channels; i++) tasklet_kill(&bdev->channels[i].vc.task); @@ -1410,7 +1412,7 @@ static void bam_dma_remove(struct platform_device *pdev) /* mask all interrupts for this execution environment */ writel_relaxed(0, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); - devm_free_irq(bdev->dev, bdev->irq, bdev); + free_irq(bdev->irq, bdev); for (i = 0; i < bdev->num_channels; i++) { bam_dma_terminate_all(&bdev->channels[i].vc.chan); diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index a5055a6273af..c41bfac65ddf 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -1965,16 +1965,19 @@ static int gpi_ch_init(struct gchan *gchan) error_start_chan: for (i = i - 1; i >= 0; i--) { gpi_stop_chan(&gpii->gchan[i]); - gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET); + gpi_send_cmd(gpii, &gpii->gchan[i], GPI_CH_CMD_RESET); } i = 2; error_alloc_chan: for (i = i - 1; i >= 0; i--) - gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC); + gpi_reset_chan(&gpii->gchan[i], GPI_CH_CMD_DE_ALLOC); error_alloc_ev_ring: gpi_disable_interrupts(gpii); error_config_int: gpi_free_ring(&gpii->ev_ring, gpii); + write_lock_irq(&gpii->pm_lock); + gpii->pm_state = DISABLE_STATE; + write_unlock_irq(&gpii->pm_lock); exit_gpi_init: return ret; } @@ -2065,6 +2068,8 @@ static int gpi_alloc_chan_resources(struct dma_chan *chan) goto xfer_alloc_err; ret = gpi_ch_init(gchan); + if (ret) + gpi_free_ring(&gchan->ch_ring, gpii); mutex_unlock(&gpii->ctrl_lock); diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index c939635be21d..1349066b1544 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -935,10 +935,10 @@ static void hidma_remove(struct platform_device *pdev) #if IS_ENABLED(CONFIG_ACPI) static const struct acpi_device_id hidma_acpi_ids[] = { - {"QCOM8061"}, - {"QCOM8062", HIDMA_MSI_CAP}, - {"QCOM8063", (HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP)}, - {}, + { .id = "QCOM8061" }, + { .id = "QCOM8062", .driver_data = HIDMA_MSI_CAP }, + { .id = "QCOM8063", .driver_data = (HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP) }, + { } }; MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids); #endif diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 44eab2d21d54..3ad8aba5dcd4 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -219,10 +219,12 @@ struct rcar_dmac { * struct rcar_dmac_of_data - This driver's OF data * @chan_offset_base: DMAC channels base offset * @chan_offset_stride: DMAC channels offset stride + * @broken_ipmmu_microTLB0: flag if the connected IPMMU has microTLB0 issues */ struct rcar_dmac_of_data { u32 chan_offset_base; u32 chan_offset_stride; + unsigned int broken_ipmmu_microTLB0:1; }; /* ----------------------------------------------------------------------------- @@ -1877,8 +1879,9 @@ static int rcar_dmac_probe(struct platform_device *pdev) * can't use it with the IPMMU. As the IOMMU API operates at the device * level we can't disable it selectively, so ignore channel 0 for now if * the device is part of an IOMMU group. + * Not applicable if system uses something else than IPMMU like SMMU. */ - if (device_iommu_mapped(&pdev->dev)) + if (data->broken_ipmmu_microTLB0 && device_iommu_mapped(&pdev->dev)) dmac->channels_mask &= ~BIT(0); dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, @@ -1998,13 +2001,21 @@ static void rcar_dmac_shutdown(struct platform_device *pdev) } static const struct rcar_dmac_of_data rcar_dmac_data = { - .chan_offset_base = 0x8000, - .chan_offset_stride = 0x80, + .chan_offset_base = 0x8000, + .chan_offset_stride = 0x80, + .broken_ipmmu_microTLB0 = 1, }; static const struct rcar_dmac_of_data rcar_gen4_dmac_data = { - .chan_offset_base = 0x0, - .chan_offset_stride = 0x1000, + .chan_offset_base = 0x0, + .chan_offset_stride = 0x1000, + /* Set to play safe. Needs testing if Gen4 is really affected */ + .broken_ipmmu_microTLB0 = 1, +}; + +static const struct rcar_dmac_of_data rcar_gen5_dmac_data = { + .chan_offset_base = 0x0, + .chan_offset_stride = 0x1000, }; static const struct of_device_id rcar_dmac_of_ids[] = { @@ -2014,6 +2025,9 @@ static const struct of_device_id rcar_dmac_of_ids[] = { }, { .compatible = "renesas,rcar-gen4-dmac", .data = &rcar_gen4_dmac_data, + }, { + .compatible = "renesas,rcar-gen5-dmac", + .data = &rcar_gen5_dmac_data, }, { /* * Backward compatibility for between v5.12 - v5.19 @@ -2039,6 +2053,6 @@ static struct platform_driver rcar_dmac_driver = { module_platform_driver(rcar_dmac_driver); -MODULE_DESCRIPTION("R-Car Gen2 DMA Controller Driver"); +MODULE_DESCRIPTION("R-Car DMA Controller Driver"); MODULE_AUTHOR("Laurent Pinchart "); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c index ca76f1bb45c4..fdd7131e890a 100644 --- a/drivers/dma/sh/rz-dmac.c +++ b/drivers/dma/sh/rz-dmac.c @@ -937,7 +937,7 @@ static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, struct rz_dmac *dmac = to_rz_dmac(chan->device); u32 residue = 0, i = 0; - while (lmdesc->nxla != crla) { + while (rz_dmac_lmdesc_addr(channel, lmdesc) != crla) { lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); if (++i >= DMAC_NR_LMDESC) return 0; @@ -948,13 +948,13 @@ static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, u32 start_lmdesc_addr = rz_dmac_lmdesc_addr(channel, desc->start_lmdesc); while (lmdesc->nxla != start_lmdesc_addr) { - residue += lmdesc->tb; lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); + residue += lmdesc->tb; } } else { while (lmdesc->chcfg & CHCFG_DEM) { - residue += lmdesc->tb; lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); + residue += lmdesc->tb; } } diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c index 19b32a23c882..60b3067a0099 100644 --- a/drivers/dma/sprd-dma.c +++ b/drivers/dma/sprd-dma.c @@ -208,8 +208,7 @@ struct sprd_dma_chn { struct sprd_dma_dev { struct dma_device dma_dev; void __iomem *glb_base; - struct clk *clk; - struct clk *ashb_clk; + struct clk_bulk_data clks[2]; int irq; u32 total_chns; struct sprd_dma_chn channels[] __counted_by(total_chns); @@ -260,31 +259,12 @@ static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg, static int sprd_dma_enable(struct sprd_dma_dev *sdev) { - int ret; - - ret = clk_prepare_enable(sdev->clk); - if (ret) - return ret; - - /* - * The ashb_clk is optional and only for AGCP DMA controller, so we - * need add one condition to check if the ashb_clk need enable. - */ - if (!IS_ERR(sdev->ashb_clk)) - ret = clk_prepare_enable(sdev->ashb_clk); - - return ret; + return clk_bulk_prepare_enable(ARRAY_SIZE(sdev->clks), sdev->clks); } static void sprd_dma_disable(struct sprd_dma_dev *sdev) { - clk_disable_unprepare(sdev->clk); - - /* - * Need to check if we need disable the optional ashb_clk for AGCP DMA. - */ - if (!IS_ERR(sdev->ashb_clk)) - clk_disable_unprepare(sdev->ashb_clk); + clk_bulk_disable_unprepare(ARRAY_SIZE(sdev->clks), sdev->clks); } static void sprd_dma_set_uid(struct sprd_dma_chn *schan) @@ -1140,16 +1120,17 @@ static int sprd_dma_probe(struct platform_device *pdev) if (!sdev) return -ENOMEM; - sdev->clk = devm_clk_get(&pdev->dev, "enable"); - if (IS_ERR(sdev->clk)) { - dev_err(&pdev->dev, "get enable clock failed\n"); - return PTR_ERR(sdev->clk); - } + /* The ashb_eb clock is optional and only present for AGCP DMA */ + sdev->clks[0].id = "enable"; + sdev->clks[1].id = "ashb_eb"; + ret = devm_clk_bulk_get_optional(&pdev->dev, ARRAY_SIZE(sdev->clks), + sdev->clks); + if (ret) + return dev_err_probe(&pdev->dev, ret, "get clocks failed\n"); - /* ashb clock is optional for AGCP DMA */ - sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb"); - if (IS_ERR(sdev->ashb_clk)) - dev_warn(&pdev->dev, "no optional ashb eb clock\n"); + if (!sdev->clks[0].clk) + return dev_err_probe(&pdev->dev, -ENOENT, + "get enable clock failed\n"); /* * We have three DMA controllers: AP DMA, AON DMA and AGCP DMA. For AGCP diff --git a/drivers/dma/st_fdma.c b/drivers/dma/st_fdma.c index d9547017f3bd..35dbff5f9f1a 100644 --- a/drivers/dma/st_fdma.c +++ b/drivers/dma/st_fdma.c @@ -268,7 +268,7 @@ static int st_fdma_alloc_chan_res(struct dma_chan *chan) struct st_fdma_chan *fchan = to_st_fdma_chan(chan); /* Create the dma pool for descriptor allocation */ - fchan->node_pool = dma_pool_create(dev_name(&chan->dev->device), + fchan->node_pool = dma_pool_create(dma_chan_name(chan), fchan->fdev->dev, sizeof(struct st_fdma_hw_node), __alignof__(struct st_fdma_hw_node), diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 0d9ffa3e2663..74dd4cbdc3c0 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -28,6 +28,8 @@ #include "ste_dma40.h" #include "ste_dma40_ll.h" +#define D40_MEMCPY_MAX_CHANS 8 + /** * struct stedma40_platform_data - Configuration struct for the dma device. * @@ -41,6 +43,7 @@ * to use SoftLLI. * @use_esram_lcla: flag for mapping the lcla into esram region * @num_of_memcpy_chans: The number of channels reserved for memcpy. + * @memcpy_channels: The event lines used for memcpy. * @num_of_phy_chans: The number of physical channels implemented in HW. * 0 means reading the number of channels from DMA HW but this is only valid * for 'multiple of 4' channels, like 8. @@ -51,6 +54,7 @@ struct stedma40_platform_data { int num_of_soft_lli_chans; bool use_esram_lcla; int num_of_memcpy_chans; + u32 memcpy_channels[D40_MEMCPY_MAX_CHANS]; int num_of_phy_chans; }; @@ -65,6 +69,9 @@ struct stedma40_platform_data { /* Maximum iterations taken before giving up suspending a channel */ #define D40_SUSPEND_MAX_IT 500 +/* Maximum attempts to sample a stable cyclic residue position */ +#define D40_RESIDUE_MAX_ATTEMPTS 3 + /* Milliseconds */ #define DMA40_AUTOSUSPEND_DELAY 100 @@ -75,6 +82,11 @@ struct stedma40_platform_data { #define D40_LCLA_LINK_PER_EVENT_GRP 128 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP +/* Number of event groups per hardware register layout */ +#define D40_EVENT_GROUPS_V4A 4 +#define D40_EVENT_GROUPS_V4B 5 +#define D40_PHYS_PER_EVENT_GROUP 2 + /* Max number of logical channels per physical channel */ #define D40_MAX_LOG_CHAN_PER_PHY 32 @@ -86,25 +98,6 @@ struct stedma40_platform_data { #define D40_ALLOC_PHY BIT(30) #define D40_ALLOC_LOG_FREE 0 -#define D40_MEMCPY_MAX_CHANS 8 - -/* Reserved event lines for memcpy only. */ -#define DB8500_DMA_MEMCPY_EV_0 51 -#define DB8500_DMA_MEMCPY_EV_1 56 -#define DB8500_DMA_MEMCPY_EV_2 57 -#define DB8500_DMA_MEMCPY_EV_3 58 -#define DB8500_DMA_MEMCPY_EV_4 59 -#define DB8500_DMA_MEMCPY_EV_5 60 - -static int dma40_memcpy_channels[] = { - DB8500_DMA_MEMCPY_EV_0, - DB8500_DMA_MEMCPY_EV_1, - DB8500_DMA_MEMCPY_EV_2, - DB8500_DMA_MEMCPY_EV_3, - DB8500_DMA_MEMCPY_EV_4, - DB8500_DMA_MEMCPY_EV_5, -}; - /* Default configuration for physical memcpy */ static const struct stedma40_chan_cfg dma40_memcpy_conf_phy = { .mode = STEDMA40_MODE_PHYSICAL, @@ -378,11 +371,17 @@ struct d40_lli_pool { * @lli_len: Number of llis of current descriptor. * @lli_current: Number of transferred llis. * @lcla_alloc: Number of LCLA entries allocated. + * @cyclic_dma_addr: Start address of the cyclic buffer. + * @cyclic_buf_len: Length of the cyclic buffer. + * @cyclic_residue: Last valid cyclic residue sample. + * @cyclic_period_len: Length of one cyclic period. + * @cyclic_callback_pos: Position after the callbacks already queued. * @txd: DMA engine struct. Used for among other things for communication * during a transfer. * @node: List entry. * @is_in_client_list: true if the client owns this descriptor. * @cyclic: true if this is a cyclic job + * @cyclic_callback_pos_valid: Whether cyclic_callback_pos is reliable. * * This descriptor is used for both logical and physical transfers. */ @@ -396,12 +395,18 @@ struct d40_desc { int lli_len; int lli_current; int lcla_alloc; + dma_addr_t cyclic_dma_addr; + size_t cyclic_buf_len; + size_t cyclic_residue; + size_t cyclic_period_len; + size_t cyclic_callback_pos; struct dma_async_tx_descriptor txd; struct list_head node; bool is_in_client_list; bool cyclic; + bool cyclic_callback_pos_valid; }; /** @@ -411,8 +416,8 @@ struct d40_desc { * @dma_addr: DMA address, if mapped * @base_unaligned: The original kmalloc pointer, if kmalloc is used. * This pointer is only there for clean-up on error. - * @pages: The number of pages needed for all physical channels. - * Only used later for clean-up on error + * @alloc_order: Order used for the LCLA page allocation. + * Only used later for clean-up on error. * @lock: Lock to protect the content in this struct. * @alloc_map: big map over which LCLA entry is own by which job. */ @@ -420,7 +425,7 @@ struct d40_lcla_pool { void *base; dma_addr_t dma_addr; void *base_unaligned; - int pages; + unsigned int alloc_order; spinlock_t lock; struct d40_desc **alloc_map; }; @@ -504,7 +509,7 @@ struct d40_chan { u32 src_def_cfg; u32 dst_def_cfg; struct d40_def_lcsp log_def; - struct d40_log_lli_full *lcpa; + struct d40_log_lli_full __iomem *lcpa; /* Runtime reconfiguration */ dma_addr_t runtime_addr; enum dma_transfer_direction runtime_direction; @@ -522,6 +527,7 @@ struct d40_chan { * @high_prio_clear: the high priority clear register * @interrupt_en: the interrupt enable register * @interrupt_clear: the interrupt clear register + * @num_event_groups: number of supported event groups * @il: the pointer to struct d40_interrupt_lookup * @il_size: the size of d40_interrupt_lookup array * @init_reg: the pointer to the struct d40_reg_val @@ -536,6 +542,7 @@ struct d40_gen_dmac { u32 high_prio_clear; u32 interrupt_en; u32 interrupt_clear; + u32 num_event_groups; struct d40_interrupt_lookup *il; u32 il_size; struct d40_reg_val *init_reg; @@ -610,7 +617,7 @@ struct d40_base { /* Physical half channels */ struct d40_phy_res *phy_res; struct d40_lcla_pool lcla_pool; - void *lcpa_base; + void __iomem *lcpa_base; dma_addr_t phy_lcpa; resource_size_t lcpa_size; struct kmem_cache *desc_slab; @@ -623,11 +630,6 @@ struct d40_base { struct d40_chan phy_chans[]; }; -static struct device *chan2dev(struct d40_chan *d40c) -{ - return &d40c->chan.dev->device; -} - static bool chan_is_physical(struct d40_chan *chan) { return chan->log_num == D40_PHY_CHAN; @@ -648,7 +650,7 @@ static void __iomem *chan_base(struct d40_chan *chan) dev_err(dev, "[%s] " format, __func__, ## arg) #define chan_err(d40c, format, arg...) \ - d40_err(chan2dev(d40c), format, ## arg) + d40_err(dmaengine_chan_dev(&d40c->chan), format, ## arg) static int d40_set_runtime_config_write(struct dma_chan *chan, struct dma_slave_config *config, @@ -1243,7 +1245,7 @@ static void __d40_config_set_event(struct d40_chan *d40c, } if (tries != 99) - dev_dbg(chan2dev(d40c), + dev_dbg(dmaengine_chan_dev(&d40c->chan), "[%s] workaround enable S%cLNK (%d tries)\n", __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D', 100 - tries); @@ -1420,6 +1422,119 @@ static u32 d40_residue(struct d40_chan *d40c) return num_elt * d40c->dma_cfg.dst_info.data_width; } +static bool d40_current_addr(struct d40_chan *d40c, dma_addr_t *addr) +{ + bool dst = d40c->dma_cfg.dir == DMA_DEV_TO_MEM; + void __iomem *high_reg; + void __iomem *low_reg; + u32 low; + u32 high; + u32 check; + int i; + + if (chan_is_physical(d40c)) { + *addr = readl(chan_base(d40c) + + (dst ? D40_CHAN_REG_SDPTR : D40_CHAN_REG_SSPTR)); + return true; + } + + if (dst) { + low_reg = &d40c->lcpa->lcsp2; + high_reg = &d40c->lcpa->lcsp3; + } else { + low_reg = &d40c->lcpa->lcsp0; + high_reg = &d40c->lcpa->lcsp1; + } + + for (i = 0; i < D40_RESIDUE_MAX_ATTEMPTS; i++) { + high = readl(high_reg) & D40_MEM_LCSP1_SPTR_MASK; + low = readl(low_reg) & D40_MEM_LCSP0_SPTR_MASK; + check = readl(high_reg) & D40_MEM_LCSP1_SPTR_MASK; + if (high == check) { + *addr = low | high; + return true; + } + } + + return false; +} + +static bool d40_cyclic_offset(struct d40_chan *d40c, struct d40_desc *d40d, + size_t *offset) +{ + dma_addr_t current_addr; + dma_addr_t current_offset; + int i; + + for (i = 0; i < D40_RESIDUE_MAX_ATTEMPTS; i++) { + if (!d40_current_addr(d40c, ¤t_addr)) + continue; + + current_offset = current_addr - d40d->cyclic_dma_addr; + if (current_offset <= d40d->cyclic_buf_len) { + *offset = current_offset; + return true; + } + } + + return false; +} + +static unsigned int d40_cyclic_periods_elapsed(struct d40_chan *d40c, + struct d40_desc *d40d) +{ + size_t current_pos; + size_t delta; + size_t offset; + unsigned int periods; + + if (!d40d->cyclic_period_len) + return 1; + + if (!d40_cyclic_offset(d40c, d40d, &offset)) { + d40d->cyclic_callback_pos_valid = false; + return 1; + } + + current_pos = rounddown(offset, d40d->cyclic_period_len); + if (!d40_residue(d40c) && current_pos != offset) + current_pos += d40d->cyclic_period_len; + if (current_pos == d40d->cyclic_buf_len) + current_pos = 0; + + if (!d40d->cyclic_callback_pos_valid) { + /* + * The previous interrupt was reported without a pointer + * sample. Resynchronize without using the stale position, + * which would count that callback again. + */ + d40d->cyclic_callback_pos = current_pos; + d40d->cyclic_callback_pos_valid = true; + return 1; + } + + /* + * The pointer wraps with the cyclic buffer, so its displacement is + * only the minimum number of elapsed periods. Complete buffer laps + * are not observable. + */ + delta = (current_pos + d40d->cyclic_buf_len - + d40d->cyclic_callback_pos) % d40d->cyclic_buf_len; + periods = delta / d40d->cyclic_period_len; + if (!periods) { + /* + * The TC status is a single latched bit. An unchanged pointer + * cannot distinguish a complete lap from a repeated interrupt, + * so do not amplify it into a buffer's worth of callbacks. + */ + periods = 1; + } + + d40d->cyclic_callback_pos = current_pos; + + return periods; +} + static bool d40_tx_is_linked(struct d40_chan *d40c) { bool is_link; @@ -1448,11 +1563,14 @@ static int d40_pause(struct dma_chan *chan) return 0; spin_lock_irqsave(&d40c->lock, flags); - pm_runtime_get_sync(d40c->base->dev); + res = pm_runtime_resume_and_get(d40c->base->dev); + if (res < 0) + goto unlock; res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); pm_runtime_put_autosuspend(d40c->base->dev); + unlock: spin_unlock_irqrestore(&d40c->lock, flags); return res; } @@ -1472,13 +1590,16 @@ static int d40_resume(struct dma_chan *chan) return 0; spin_lock_irqsave(&d40c->lock, flags); - pm_runtime_get_sync(d40c->base->dev); + res = pm_runtime_resume_and_get(d40c->base->dev); + if (res < 0) + goto unlock; /* If bytes left to transfer or linked tx resume job */ if (d40_residue(d40c) || d40_tx_is_linked(d40c)) res = d40_channel_execute_command(d40c, D40_DMA_RUN); pm_runtime_put_autosuspend(d40c->base->dev); + unlock: spin_unlock_irqrestore(&d40c->lock, flags); return res; } @@ -1515,8 +1636,20 @@ static struct d40_desc *d40_queue_start(struct d40_chan *d40c) if (d40d != NULL) { if (!d40c->busy) { + err = pm_runtime_resume_and_get(d40c->base->dev); + if (err < 0) { + chan_err(d40c, "Failed to resume DMA: %d\n", + err); + do { + d40_desc_remove(d40d); + d40_desc_done(d40c, d40d); + d40c->pending_tx++; + d40d = d40_first_queued(d40c); + } while (d40d); + tasklet_schedule(&d40c->tasklet); + return ERR_PTR(err); + } d40c->busy = true; - pm_runtime_get_sync(d40c->base->dev); } /* Remove from queue */ @@ -1531,8 +1664,15 @@ static struct d40_desc *d40_queue_start(struct d40_chan *d40c) /* Start dma job */ err = d40_start(d40c); - if (err) - return NULL; + if (err) { + d40_desc_remove(d40d); + d40_desc_done(d40c, d40d); + d40c->pending_tx++; + d40c->busy = false; + pm_runtime_put_autosuspend(d40c->base->dev); + tasklet_schedule(&d40c->tasklet); + return ERR_PTR(err); + } } return d40d; @@ -1542,6 +1682,7 @@ static struct d40_desc *d40_queue_start(struct d40_chan *d40c) static void dma_tc_handle(struct d40_chan *d40c) { struct d40_desc *d40d; + unsigned int callbacks = 1; /* Get first active entry from list */ d40d = d40_first_active_get(d40c); @@ -1566,6 +1707,8 @@ static void dma_tc_handle(struct d40_chan *d40c) if (d40d->lli_current == d40d->lli_len) d40d->lli_current = 0; } + + callbacks = d40_cyclic_periods_elapsed(d40c, d40d); } else { d40_lcla_free_all(d40c, d40d); @@ -1576,17 +1719,17 @@ static void dma_tc_handle(struct d40_chan *d40c) return; } + d40_desc_remove(d40d); + d40_desc_done(d40c, d40d); + if (d40_queue_start(d40c) == NULL) { d40c->busy = false; pm_runtime_put_autosuspend(d40c->base->dev); } - - d40_desc_remove(d40d); - d40_desc_done(d40c, d40d); } - d40c->pending_tx++; + d40c->pending_tx += callbacks; tasklet_schedule(&d40c->tasklet); } @@ -1597,20 +1740,22 @@ static void dma_tasklet(struct tasklet_struct *t) struct d40_desc *d40d; unsigned long flags; bool callback_active; + bool from_done; struct dmaengine_desc_callback cb; spin_lock_irqsave(&d40c->lock, flags); /* Get first entry from the done list */ d40d = d40_first_done(d40c); - if (d40d == NULL) { + from_done = !!d40d; + if (!from_done) { /* Check if we have reached here for cyclic job */ d40d = d40_first_active_get(d40c); if (d40d == NULL || !d40d->cyclic) goto check_pending_tx; } - if (!d40d->cyclic) + if (from_done) dma_cookie_complete(&d40d->txd); /* @@ -1626,7 +1771,7 @@ static void dma_tasklet(struct tasklet_struct *t) callback_active = !!(d40d->txd.flags & DMA_PREP_INTERRUPT); dmaengine_desc_get_callback(&d40d->txd, &cb); - if (!d40d->cyclic) { + if (from_done) { if (async_tx_test_ack(&d40d->txd)) { d40_desc_remove(d40d); d40_desc_free(d40c, d40d); @@ -1658,6 +1803,7 @@ static void dma_tasklet(struct tasklet_struct *t) static irqreturn_t d40_handle_interrupt(int irq, void *data) { + irqreturn_t handled = IRQ_NONE; int i; u32 idx; u32 row; @@ -1667,6 +1813,11 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data) u32 *regs = base->regs_interrupt; struct d40_interrupt_lookup *il = base->gen_dmac.il; u32 il_size = base->gen_dmac.il_size; + int ret; + + ret = pm_runtime_get_if_active(base->dev); + if (IS_ENABLED(CONFIG_PM) && ret <= 0) + return IRQ_NONE; spin_lock(&base->interrupt_lock); @@ -1686,6 +1837,12 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data) row = chan / BITS_PER_LONG; idx = chan & (BITS_PER_LONG - 1); + /* + * Status for a channel owned by another core still explains + * the interrupt, but only ACK Linux-owned channels below. + */ + handled = IRQ_HANDLED; + if (il[row].offset == D40_PHY_CHAN) d40c = base->lookup_phy_chans[idx]; else @@ -1715,7 +1872,10 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data) spin_unlock(&base->interrupt_lock); - return IRQ_HANDLED; + if (ret > 0) + pm_runtime_put_autosuspend(base->dev); + + return handled; } static int d40_validate_conf(struct d40_chan *d40c, @@ -1723,19 +1883,43 @@ static int d40_validate_conf(struct d40_chan *d40c, { int res = 0; bool is_log = conf->mode == STEDMA40_MODE_LOGICAL; + bool invalid_dev_type = conf->dev_type < 0; - if (!conf->dir) { + if (!invalid_dev_type && + D40_TYPE_TO_GROUP(conf->dev_type) >= + d40c->base->gen_dmac.num_event_groups) + invalid_dev_type = true; + + if (conf->dir != DMA_MEM_TO_MEM && + !is_slave_direction(conf->dir)) { chan_err(d40c, "Invalid direction.\n"); res = -EINVAL; } - if ((is_log && conf->dev_type > d40c->base->num_log_chans) || - (!is_log && conf->dev_type > d40c->base->num_phy_chans) || - (conf->dev_type < 0)) { + if (!invalid_dev_type && is_log) { + int max_dev_type; + + if (conf->dir == DMA_DEV_TO_MEM) + max_dev_type = DIV_ROUND_UP(d40c->base->num_log_chans, 2); + else + max_dev_type = d40c->base->num_log_chans / 2; + + invalid_dev_type = conf->dev_type >= max_dev_type; + } + + if (invalid_dev_type) { chan_err(d40c, "Invalid device type (%d)\n", conf->dev_type); res = -EINVAL; } + if (conf->use_fixed_channel && + (conf->phy_channel < 0 || + conf->phy_channel >= d40c->base->num_phy_chans)) { + chan_err(d40c, "Invalid physical channel (%d)\n", + conf->phy_channel); + res = -EINVAL; + } + if (conf->dir == DMA_DEV_TO_DEV) { /* * DMAC HW supports it. Will be added to this driver, @@ -1848,6 +2032,12 @@ static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src, return is_free; } +static int d40_phy_to_group(struct d40_base *base, int phy) +{ + return (phy / D40_PHYS_PER_EVENT_GROUP) % + base->gen_dmac.num_event_groups; +} + static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user) { int dev_type = d40c->dma_cfg.dev_type; @@ -1858,11 +2048,14 @@ static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user) int j; int log_num; int num_phy_chans; + int phy_group_stride; bool is_src; bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL; phys = d40c->base->phy_res; num_phy_chans = d40c->base->num_phy_chans; + phy_group_stride = D40_PHYS_PER_EVENT_GROUP * + d40c->base->gen_dmac.num_event_groups; if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM) { log_num = 2 * dev_type; @@ -1896,9 +2089,18 @@ static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user) } } } else - for (j = 0; j < d40c->base->num_phy_chans; j += 8) { - int phy_num = j + event_group * 2; - for (i = phy_num; i < phy_num + 2; i++) { + for (j = 0; j < num_phy_chans; + j += phy_group_stride) { + int phy_num = j + event_group * + D40_PHYS_PER_EVENT_GROUP; + + if (phy_num + D40_PHYS_PER_EVENT_GROUP > + num_phy_chans) + break; + + for (i = phy_num; + i < phy_num + D40_PHYS_PER_EVENT_GROUP; + i++) { if (d40_alloc_mask_set(&phys[i], is_src, 0, @@ -1917,23 +2119,23 @@ static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user) return -EINVAL; /* Find logical channel */ - for (j = 0; j < d40c->base->num_phy_chans; j += 8) { - int phy_num = j + event_group * 2; + for (j = 0; j < num_phy_chans; j += phy_group_stride) { + int phy_num = j + event_group * D40_PHYS_PER_EVENT_GROUP; + + if (phy_num + D40_PHYS_PER_EVENT_GROUP > num_phy_chans) + break; if (d40c->dma_cfg.use_fixed_channel) { i = d40c->dma_cfg.phy_channel; - if ((i != phy_num) && (i != phy_num + 1)) { - dev_err(chan2dev(d40c), - "invalid fixed phy channel %d\n", i); - return -EINVAL; - } + if (i != phy_num && i != phy_num + 1) + continue; if (d40_alloc_mask_set(&phys[i], is_src, event_line, is_log, first_phy_user)) goto found_log; - dev_err(chan2dev(d40c), + dev_err(dmaengine_chan_dev(&d40c->chan), "could not allocate fixed phy channel %d\n", i); return -EINVAL; } @@ -1944,14 +2146,17 @@ static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user) * channels. */ if (is_src) { - for (i = phy_num; i < phy_num + 2; i++) { + for (i = phy_num; + i < phy_num + D40_PHYS_PER_EVENT_GROUP; + i++) { if (d40_alloc_mask_set(&phys[i], is_src, event_line, is_log, first_phy_user)) goto found_log; } } else { - for (i = phy_num + 1; i >= phy_num; i--) { + for (i = phy_num + D40_PHYS_PER_EVENT_GROUP - 1; + i >= phy_num; i--) { if (d40_alloc_mask_set(&phys[i], is_src, event_line, is_log, first_phy_user)) @@ -1978,10 +2183,16 @@ static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user) static int d40_config_memcpy(struct d40_chan *d40c) { dma_cap_mask_t cap = d40c->chan.device->cap_mask; + int ret; if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) { d40c->dma_cfg = dma40_memcpy_conf_log; - d40c->dma_cfg.dev_type = dma40_memcpy_channels[d40c->chan.chan_id]; + d40c->dma_cfg.dev_type = + d40c->base->plat_data->memcpy_channels[d40c->chan.chan_id]; + + ret = d40_validate_conf(d40c, &d40c->dma_cfg); + if (ret) + return ret; d40_log_cfg(&d40c->dma_cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); @@ -1990,6 +2201,10 @@ static int d40_config_memcpy(struct d40_chan *d40c) dma_has_cap(DMA_SLAVE, cap)) { d40c->dma_cfg = dma40_memcpy_conf_phy; + ret = d40_validate_conf(d40c, &d40c->dma_cfg); + if (ret) + return ret; + /* Generate interrupt at end of transfer or relink. */ d40c->dst_def_cfg |= BIT(D40_SREG_CFG_TIM_POS); @@ -2013,9 +2228,6 @@ static int d40_free_dma(struct d40_chan *d40c) struct d40_phy_res *phy = d40c->phy_chan; bool is_src; - /* Terminate all queued and active transfers */ - d40_term_all(d40c); - if (phy == NULL) { chan_err(d40c, "phy == null\n"); return -EINVAL; @@ -2037,11 +2249,18 @@ static int d40_free_dma(struct d40_chan *d40c) return -EINVAL; } - pm_runtime_get_sync(d40c->base->dev); - res = d40_channel_execute_command(d40c, D40_DMA_STOP); - if (res) { - chan_err(d40c, "stop failed\n"); - goto mark_last_busy; + /* Release descriptor state; this does not access DMA40 registers. */ + d40_term_all(d40c); + + res = pm_runtime_resume_and_get(d40c->base->dev); + if (res >= 0) { + res = d40_channel_execute_command(d40c, D40_DMA_STOP); + if (res) + chan_err(d40c, "stop failed\n"); + + pm_runtime_put_autosuspend(d40c->base->dev); + if (res) + return res; } d40_alloc_mask_free(phy, is_src, chan_is_logical(d40c) ? event : 0); @@ -2057,8 +2276,6 @@ static int d40_free_dma(struct d40_chan *d40c) d40c->busy = false; d40c->phy_chan = NULL; d40c->configured = false; - mark_last_busy: - pm_runtime_put_autosuspend(d40c->base->dev); return res; } @@ -2108,15 +2325,26 @@ static bool d40_is_paused(struct d40_chan *d40c) } -static u32 stedma40_residue(struct dma_chan *chan) +static u32 stedma40_residue(struct dma_chan *chan, dma_cookie_t cookie) { struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); + struct d40_desc *d40d; + size_t offset; u32 bytes_left; unsigned long flags; spin_lock_irqsave(&d40c->lock, flags); - bytes_left = d40_residue(d40c); + d40d = d40_first_active_get(d40c); + if (d40d && d40d->txd.cookie == cookie && d40d->cyclic && + d40d->cyclic_buf_len) { + if (d40_cyclic_offset(d40c, d40d, &offset)) + d40d->cyclic_residue = d40d->cyclic_buf_len - offset; + bytes_left = d40d->cyclic_residue; + } else { + bytes_left = d40_residue(d40c); + } + spin_unlock_irqrestore(&d40c->lock, flags); return bytes_left; @@ -2238,7 +2466,13 @@ d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src, return NULL; } - d40_set_runtime_config_write(dchan, &chan->slave_config, direction); + if (direction != DMA_MEM_TO_MEM) { + ret = d40_set_runtime_config_write(dchan, + &chan->slave_config, + direction); + if (ret) + return NULL; + } spin_lock_irqsave(&chan->lock, flags); @@ -2246,8 +2480,13 @@ d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src, if (desc == NULL) goto unlock; - if (sg_next(&sg_src[sg_len - 1]) == sg_src) + if (sg_next(&sg_src[sg_len - 1]) == sg_src) { desc->cyclic = true; + if (desc->lli_len != sg_len) { + chan_err(chan, "Cyclic periods must fit in one LLI\n"); + goto free_desc; + } + } src_dev_addr = 0; dst_dev_addr = 0; @@ -2365,6 +2604,9 @@ static struct dma_chan *d40_xlate(struct of_phandle_args *dma_spec, dma_cap_mask_t cap; u32 flags; + if (dma_spec->args_count != 3) + return NULL; + memset(&cfg, 0, sizeof(struct stedma40_chan_cfg)); dma_cap_zero(cap); @@ -2417,10 +2659,14 @@ static int d40_alloc_chan_resources(struct dma_chan *chan) err = d40_config_memcpy(d40c); if (err) { chan_err(d40c, "Failed to configure memcpy channel\n"); - goto mark_last_busy; + goto unlock; } } + err = pm_runtime_resume_and_get(d40c->base->dev); + if (err < 0) + goto unlock; + err = d40_allocate_channel(d40c, &is_free_phy); if (err) { chan_err(d40c, "Failed to allocate channel\n"); @@ -2428,8 +2674,6 @@ static int d40_alloc_chan_resources(struct dma_chan *chan) goto mark_last_busy; } - pm_runtime_get_sync(d40c->base->dev); - d40_set_prio_realtime(d40c); if (chan_is_logical(d40c)) { @@ -2446,7 +2690,7 @@ static int d40_alloc_chan_resources(struct dma_chan *chan) d40c->dst_def_cfg |= BIT(D40_SREG_CFG_LOG_GIM_POS); } - dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n", + dev_dbg(dmaengine_chan_dev(&d40c->chan), "allocated %s channel (phy %d%s)\n", chan_is_logical(d40c) ? "logical" : "physical", d40c->phy_chan->num, d40c->dma_cfg.use_fixed_channel ? ", fixed" : ""); @@ -2461,6 +2705,7 @@ static int d40_alloc_chan_resources(struct dma_chan *chan) d40_config_write(d40c); mark_last_busy: pm_runtime_put_autosuspend(d40c->base->dev); + unlock: spin_unlock_irqrestore(&d40c->lock, flags); return err; } @@ -2524,11 +2769,18 @@ dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, unsigned long flags) { - unsigned int periods = buf_len / period_len; + unsigned int periods; struct dma_async_tx_descriptor *txd; + struct d40_desc *desc; struct scatterlist *sg; + dma_addr_t buf_addr = dma_addr; int i; + if (!buf_len || !period_len || buf_len % period_len) + return NULL; + + periods = buf_len / period_len; + sg = kzalloc_objs(struct scatterlist, periods + 1, GFP_NOWAIT); if (!sg) return NULL; @@ -2543,6 +2795,15 @@ dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr, txd = d40_prep_sg(chan, sg, sg, periods, direction, DMA_PREP_INTERRUPT); + if (txd) { + desc = container_of(txd, struct d40_desc, txd); + desc->cyclic_dma_addr = buf_addr; + desc->cyclic_buf_len = buf_len; + desc->cyclic_residue = buf_len; + desc->cyclic_period_len = period_len; + desc->cyclic_callback_pos = 0; + desc->cyclic_callback_pos_valid = true; + } kfree(sg); @@ -2563,7 +2824,7 @@ static enum dma_status d40_tx_status(struct dma_chan *chan, ret = dma_cookie_status(chan, cookie, txstate); if (ret != DMA_COMPLETE && txstate) - dma_set_residue(txstate, stedma40_residue(chan)); + dma_set_residue(txstate, stedma40_residue(chan, cookie)); if (d40_is_paused(d40c)) ret = DMA_PAUSED; @@ -2605,19 +2866,26 @@ static int d40_terminate_all(struct dma_chan *chan) spin_lock_irqsave(&d40c->lock, flags); - pm_runtime_get_sync(d40c->base->dev); - ret = d40_channel_execute_command(d40c, D40_DMA_STOP); - if (ret) - chan_err(d40c, "Failed to stop channel\n"); + ret = pm_runtime_resume_and_get(d40c->base->dev); + if (ret >= 0) { + ret = d40_channel_execute_command(d40c, D40_DMA_STOP); + if (ret) + chan_err(d40c, "Failed to stop channel\n"); + pm_runtime_put_autosuspend(d40c->base->dev); + } + + /* + * Always release software state, even when the controller cannot + * resume. d40_term_all() does not access DMA40 registers. + */ d40_term_all(d40c); - pm_runtime_put_autosuspend(d40c->base->dev); if (d40c->busy) pm_runtime_put_autosuspend(d40c->base->dev); d40c->busy = false; spin_unlock_irqrestore(&d40c->lock, flags); - return 0; + return ret; } static int @@ -2680,6 +2948,13 @@ static int d40_set_runtime_config_write(struct dma_chan *chan, return -EINVAL; } + if (direction != cfg->dir) { + chan_err(d40c, + "transfer direction %d differs from allocated direction %d\n", + direction, cfg->dir); + return -EINVAL; + } + src_addr_width = config->src_addr_width; src_maxburst = config->src_maxburst; dst_addr_width = config->dst_addr_width; @@ -2688,13 +2963,6 @@ static int d40_set_runtime_config_write(struct dma_chan *chan, if (direction == DMA_DEV_TO_MEM) { config_addr = config->src_addr; - if (cfg->dir != DMA_DEV_TO_MEM) - dev_dbg(d40c->base->dev, - "channel was not configured for peripheral " - "to memory transfer (%d) overriding\n", - cfg->dir); - cfg->dir = DMA_DEV_TO_MEM; - /* Configure the memory side */ if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) dst_addr_width = src_addr_width; @@ -2704,13 +2972,6 @@ static int d40_set_runtime_config_write(struct dma_chan *chan, } else if (direction == DMA_MEM_TO_DEV) { config_addr = config->dst_addr; - if (cfg->dir != DMA_MEM_TO_DEV) - dev_dbg(d40c->base->dev, - "channel was not configured for memory " - "to peripheral transfer (%d) overriding\n", - cfg->dir); - cfg->dir = DMA_MEM_TO_DEV; - /* Configure the memory side */ if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) src_addr_width = dst_addr_width; @@ -2822,6 +3083,18 @@ static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma, } } +static void d40_kill_tasklets(void *data) +{ + struct dma_device *dma = data; + struct d40_chan *d40c; + struct dma_chan *chan; + + list_for_each_entry(chan, &dma->channels, device_node) { + d40c = container_of(chan, struct d40_chan, chan); + tasklet_kill(&d40c->tasklet); + } +} + static void d40_ops_init(struct d40_base *base, struct dma_device *dev) { if (dma_has_cap(DMA_SLAVE, dev->cap_mask)) { @@ -2868,6 +3141,11 @@ static int __init d40_dmaengine_init(struct d40_base *base, d40_ops_init(base, &base->dma_slave); + err = devm_add_action(base->dev, d40_kill_tasklets, + &base->dma_slave); + if (err) + goto exit; + err = dmaenginem_async_device_register(&base->dma_slave); if (err) { @@ -2883,6 +3161,11 @@ static int __init d40_dmaengine_init(struct d40_base *base, d40_ops_init(base, &base->dma_memcpy); + err = devm_add_action(base->dev, d40_kill_tasklets, + &base->dma_memcpy); + if (err) + goto exit; + err = dmaenginem_async_device_register(&base->dma_memcpy); if (err) { @@ -2897,9 +3180,15 @@ static int __init d40_dmaengine_init(struct d40_base *base, dma_cap_zero(base->dma_both.cap_mask); dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask); dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask); - dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask); + dma_cap_set(DMA_CYCLIC, base->dma_both.cap_mask); d40_ops_init(base, &base->dma_both); + + err = devm_add_action(base->dev, d40_kill_tasklets, + &base->dma_both); + if (err) + goto exit; + err = dmaenginem_async_device_register(&base->dma_both); if (err) { @@ -3047,9 +3336,9 @@ static int __init d40_phy_res_init(struct d40_base *base) base->phy_res[i].allocated_src = D40_ALLOC_PHY; base->phy_res[i].allocated_dst = D40_ALLOC_PHY; base->phy_res[i].reserved = true; - gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i), + gcc |= D40_DREG_GCC_EVTGRP_ENA(d40_phy_to_group(base, i), D40_DREG_GCC_SRC); - gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i), + gcc |= D40_DREG_GCC_EVTGRP_ENA(d40_phy_to_group(base, i), D40_DREG_GCC_DST); @@ -3069,9 +3358,9 @@ static int __init d40_phy_res_init(struct d40_base *base) base->phy_res[chan].allocated_src = D40_ALLOC_PHY; base->phy_res[chan].allocated_dst = D40_ALLOC_PHY; base->phy_res[chan].reserved = true; - gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan), + gcc |= D40_DREG_GCC_EVTGRP_ENA(d40_phy_to_group(base, chan), D40_DREG_GCC_SRC); - gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan), + gcc |= D40_DREG_GCC_EVTGRP_ENA(d40_phy_to_group(base, chan), D40_DREG_GCC_DST); num_phy_chans_avail--; } @@ -3188,13 +3477,17 @@ static int __init d40_hw_detect_init(struct platform_device *pdev, num_phy_chans = min(num_phy_chans, STEDMA40_MAX_PHYS); - /* The number of channels used for memcpy */ - if (plat_data->num_of_memcpy_chans) - num_memcpy_chans = plat_data->num_of_memcpy_chans; - else - num_memcpy_chans = ARRAY_SIZE(dma40_memcpy_channels); + for (i = 0; plat_data->disabled_channels[i] != -1; i++) { + int chan = plat_data->disabled_channels[i]; - num_memcpy_chans = min(num_memcpy_chans, D40_MEMCPY_MAX_CHANS); + if (chan < 0 || chan >= num_phy_chans) { + dev_err(dev, "Invalid disabled channel %d\n", chan); + return -EINVAL; + } + } + + /* The number of channels used for memcpy */ + num_memcpy_chans = plat_data->num_of_memcpy_chans; num_log_chans = num_phy_chans * D40_MAX_LOG_CHAN_PER_PHY; dev_info(dev, @@ -3219,6 +3512,7 @@ static int __init d40_hw_detect_init(struct platform_device *pdev, base->log_chans = &base->phy_chans[num_phy_chans]; if (base->plat_data->num_of_phy_chans == 14) { + base->gen_dmac.num_event_groups = D40_EVENT_GROUPS_V4B; base->gen_dmac.backup = d40_backup_regs_v4b; base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4B; base->gen_dmac.interrupt_en = D40_DREG_CPCMIS; @@ -3232,6 +3526,7 @@ static int __init d40_hw_detect_init(struct platform_device *pdev, base->gen_dmac.init_reg = dma_init_reg_v4b; base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4b); } else { + base->gen_dmac.num_event_groups = D40_EVENT_GROUPS_V4A; if (base->rev >= 3) { base->gen_dmac.backup = d40_backup_regs_v4a; base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4A; @@ -3358,6 +3653,7 @@ static void __init d40_hw_init(struct d40_base *base) static int __init d40_lcla_allocate(struct d40_base *base) { struct d40_lcla_pool *pool = &base->lcla_pool; + size_t lcla_size = SZ_1K * base->num_phy_chans; unsigned long *page_list; int i, j; int ret; @@ -3373,20 +3669,20 @@ static int __init d40_lcla_allocate(struct d40_base *base) if (!page_list) return -ENOMEM; - /* Calculating how many pages that are required */ - base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE; + base->lcla_pool.alloc_order = get_order(lcla_size); for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) { page_list[i] = __get_free_pages(GFP_KERNEL, - base->lcla_pool.pages); + base->lcla_pool.alloc_order); if (!page_list[i]) { - d40_err(base->dev, "Failed to allocate %d pages.\n", - base->lcla_pool.pages); + d40_err(base->dev, "Failed to allocate %zu bytes.\n", + lcla_size); ret = -ENOMEM; for (j = 0; j < i; j++) - free_pages(page_list[j], base->lcla_pool.pages); + free_pages(page_list[j], + base->lcla_pool.alloc_order); goto free_page_list; } @@ -3396,7 +3692,7 @@ static int __init d40_lcla_allocate(struct d40_base *base) } for (j = 0; j < i; j++) - free_pages(page_list[j], base->lcla_pool.pages); + free_pages(page_list[j], base->lcla_pool.alloc_order); if (i < MAX_LCLA_ALLOC_ATTEMPTS) { base->lcla_pool.base = (void *)page_list[i]; @@ -3406,10 +3702,9 @@ static int __init d40_lcla_allocate(struct d40_base *base) * alignment, try with allocating a big buffer. */ dev_warn(base->dev, - "[%s] Failed to get %d pages @ 18 bit align.\n", - __func__, base->lcla_pool.pages); - base->lcla_pool.base_unaligned = kmalloc(SZ_1K * - base->num_phy_chans + + "[%s] Failed to get %zu bytes @ 18 bit align.\n", + __func__, lcla_size); + base->lcla_pool.base_unaligned = kmalloc(lcla_size + LCLA_ALIGNMENT, GFP_KERNEL); if (!base->lcla_pool.base_unaligned) { @@ -3421,8 +3716,7 @@ static int __init d40_lcla_allocate(struct d40_base *base) LCLA_ALIGNMENT); } - pool->dma_addr = dma_map_single(base->dev, pool->base, - SZ_1K * base->num_phy_chans, + pool->dma_addr = dma_map_single(base->dev, pool->base, lcla_size, DMA_TO_DEVICE); if (dma_mapping_error(base->dev, pool->dma_addr)) { pool->dma_addr = 0; @@ -3444,6 +3738,7 @@ static int __init d40_of_probe(struct device *dev, struct stedma40_platform_data *pdata; int num_phy = 0, num_memcpy = 0, num_disabled = 0; const __be32 *list; + int ret; pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) @@ -3457,18 +3752,19 @@ static int __init d40_of_probe(struct device *dev, list = of_get_property(np, "memcpy-channels", &num_memcpy); num_memcpy /= sizeof(*list); - if (num_memcpy > D40_MEMCPY_MAX_CHANS || num_memcpy <= 0) { + if (num_memcpy > ARRAY_SIZE(pdata->memcpy_channels) || + num_memcpy <= 0) { d40_err(dev, "Invalid number of memcpy channels specified (%d)\n", num_memcpy); return -EINVAL; } + ret = of_property_read_u32_array(np, "memcpy-channels", + pdata->memcpy_channels, num_memcpy); + if (ret) + return ret; pdata->num_of_memcpy_chans = num_memcpy; - of_property_read_u32_array(np, "memcpy-channels", - dma40_memcpy_channels, - num_memcpy); - list = of_get_property(np, "disabled-channels", &num_disabled); num_disabled /= sizeof(*list); @@ -3497,6 +3793,7 @@ static int __init d40_probe(struct platform_device *pdev) struct d40_base *base; struct resource *res; struct resource res_lcpa; + void *dmaenginem_reg_group; int num_reserved_chans; u32 val; int ret; @@ -3526,6 +3823,7 @@ static int __init d40_probe(struct platform_device *pdev) } /* This is no device so read the address directly from the node */ ret = of_address_to_resource(np_lcpa, 0, &res_lcpa); + of_node_put(np_lcpa); if (ret) { dev_err(dev, "no LCPA SRAM resource\n"); goto report_failure; @@ -3585,12 +3883,6 @@ static int __init d40_probe(struct platform_device *pdev) goto destroy_cache; } - ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base); - if (ret) { - d40_err(dev, "No IRQ defined\n"); - goto destroy_cache; - } - if (base->plat_data->use_esram_lcla) { base->lcpa_regulator = regulator_get(base->dev, "lcla_esram"); @@ -3615,38 +3907,62 @@ static int __init d40_probe(struct platform_device *pdev) pm_runtime_irq_safe(base->dev); pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY); - pm_runtime_use_autosuspend(base->dev); pm_runtime_mark_last_busy(base->dev); - pm_runtime_set_active(base->dev); - pm_runtime_enable(base->dev); - ret = d40_dmaengine_init(base, num_reserved_chans); - if (ret) + ret = devm_pm_runtime_set_active_enabled(base->dev); + if (ret) { + d40_err(dev, "Failed to enable runtime PM: %d\n", ret); goto destroy_cache; + } + pm_runtime_use_autosuspend(base->dev); dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE); d40_hw_init(base); + ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base); + if (ret) { + d40_err(dev, "No IRQ defined\n"); + goto destroy_cache; + } + + dmaenginem_reg_group = devres_open_group(dev, NULL, GFP_KERNEL); + if (!dmaenginem_reg_group) { + ret = -ENOMEM; + goto release_irq; + } + + ret = d40_dmaengine_init(base, num_reserved_chans); + if (ret) + goto release_dmaenginem; + ret = of_dma_controller_register(np, d40_xlate, NULL); if (ret) { dev_err(dev, "could not register of_dma_controller\n"); - goto destroy_cache; + goto release_dmaenginem; } + devres_remove_group(dev, dmaenginem_reg_group); dev_info(base->dev, "initialized\n"); return 0; + release_dmaenginem: + free_irq(base->irq, base); + devres_release_group(dev, dmaenginem_reg_group); + goto destroy_cache; + release_irq: + free_irq(base->irq, base); destroy_cache: if (base->lcla_pool.dma_addr) dma_unmap_single(base->dev, base->lcla_pool.dma_addr, SZ_1K * base->num_phy_chans, DMA_TO_DEVICE); - if (!base->lcla_pool.base_unaligned && base->lcla_pool.base) + if (!base->plat_data->use_esram_lcla && + !base->lcla_pool.base_unaligned && base->lcla_pool.base) free_pages((unsigned long)base->lcla_pool.base, - base->lcla_pool.pages); + base->lcla_pool.alloc_order); kfree(base->lcla_pool.base_unaligned); @@ -3654,7 +3970,6 @@ static int __init d40_probe(struct platform_device *pdev) regulator_disable(base->lcpa_regulator); regulator_put(base->lcpa_regulator); } - pm_runtime_disable(base->dev); report_failure: d40_err(dev, "probe failed\n"); diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c index 4c489b126cb2..3be7f5b1abb0 100644 --- a/drivers/dma/ste_dma40_ll.c +++ b/drivers/dma/ste_dma40_ll.c @@ -333,10 +333,10 @@ static void d40_log_lli_link(struct d40_log_lli *lli_dst, (dlos << D40_MEM_LCSP1_SLOS_POS); } -void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa, - struct d40_log_lli *lli_dst, - struct d40_log_lli *lli_src, - int next, unsigned int flags) +void d40_log_lli_lcpa_write(struct d40_log_lli_full __iomem *lcpa, + struct d40_log_lli *lli_dst, + struct d40_log_lli *lli_src, + int next, unsigned int flags) { d40_log_lli_link(lli_dst, lli_src, next, flags); diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h index 2e30e9a94a1e..ca9d07649361 100644 --- a/drivers/dma/ste_dma40_ll.h +++ b/drivers/dma/ste_dma40_ll.h @@ -16,8 +16,6 @@ #define D40_TYPE_TO_GROUP(type) (type / 16) #define D40_TYPE_TO_EVENT(type) (type % 16) -#define D40_GROUP_SIZE 8 -#define D40_PHYS_TO_GROUP(phys) ((phys & (D40_GROUP_SIZE - 1)) / 2) /* Most bits of the CFG register are the same in log as in phy mode */ #define D40_SREG_CFG_MST_POS 15 @@ -457,7 +455,7 @@ int d40_log_sg_to_lli(struct scatterlist *sg, u32 lcsp13, /* src or dst*/ u32 data_width1, u32 data_width2); -void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa, +void d40_log_lli_lcpa_write(struct d40_log_lli_full __iomem *lcpa, struct d40_log_lli *lli_dst, struct d40_log_lli *lli_src, int next, unsigned int flags); diff --git a/drivers/dma/stm32/stm32-dma.c b/drivers/dma/stm32/stm32-dma.c index d3ad78562a14..4688cebaf7f7 100644 --- a/drivers/dma/stm32/stm32-dma.c +++ b/drivers/dma/stm32/stm32-dma.c @@ -248,11 +248,6 @@ static struct stm32_dma_desc *to_stm32_dma_desc(struct virt_dma_desc *vdesc) return container_of(vdesc, struct stm32_dma_desc, vdesc); } -static struct device *chan2dev(struct stm32_dma_chan *chan) -{ - return &chan->vchan.chan.dev->device; -} - static u32 stm32_dma_read(struct stm32_dma_device *dmadev, u32 reg) { return readl_relaxed(dmadev->base + reg); @@ -274,7 +269,7 @@ static int stm32_dma_get_width(struct stm32_dma_chan *chan, case DMA_SLAVE_BUSWIDTH_4_BYTES: return STM32_DMA_WORD; default: - dev_err(chan2dev(chan), "Dma bus width not supported\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Dma bus width not supported\n"); return -EINVAL; } } @@ -374,7 +369,7 @@ static int stm32_dma_get_burst(struct stm32_dma_chan *chan, u32 maxburst) case 16: return STM32_DMA_BURST_INCR16; default: - dev_err(chan2dev(chan), "Dma burst size not supported\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Dma burst size not supported\n"); return -EINVAL; } } @@ -487,7 +482,7 @@ static void stm32_dma_stop(struct stm32_dma_chan *chan) /* Clear interrupt status if it is there */ status = stm32_dma_irq_status(chan); if (status) { - dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "%s(): clearing interrupt: 0x%08x\n", __func__, status); stm32_dma_irq_clear(chan, status); } @@ -536,12 +531,12 @@ static void stm32_dma_dump_reg(struct stm32_dma_chan *chan) u32 sm1ar = stm32_dma_read(dmadev, STM32_DMA_SM1AR(chan->id)); u32 sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id)); - dev_dbg(chan2dev(chan), "SCR: 0x%08x\n", scr); - dev_dbg(chan2dev(chan), "NDTR: 0x%08x\n", ndtr); - dev_dbg(chan2dev(chan), "SPAR: 0x%08x\n", spar); - dev_dbg(chan2dev(chan), "SM0AR: 0x%08x\n", sm0ar); - dev_dbg(chan2dev(chan), "SM1AR: 0x%08x\n", sm1ar); - dev_dbg(chan2dev(chan), "SFCR: 0x%08x\n", sfcr); + dev_dbg(vchan_chan_dev(&chan->vchan), "SCR: 0x%08x\n", scr); + dev_dbg(vchan_chan_dev(&chan->vchan), "NDTR: 0x%08x\n", ndtr); + dev_dbg(vchan_chan_dev(&chan->vchan), "SPAR: 0x%08x\n", spar); + dev_dbg(vchan_chan_dev(&chan->vchan), "SM0AR: 0x%08x\n", sm0ar); + dev_dbg(vchan_chan_dev(&chan->vchan), "SM1AR: 0x%08x\n", sm1ar); + dev_dbg(vchan_chan_dev(&chan->vchan), "SFCR: 0x%08x\n", sfcr); } static void stm32_dma_sg_inc(struct stm32_dma_chan *chan) @@ -613,7 +608,7 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) reg->dma_scr |= STM32_DMA_SCR_EN; stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); - dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: started\n", &chan->vchan); } static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan) @@ -630,12 +625,12 @@ static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan) if (dma_scr & STM32_DMA_SCR_CT) { dma_sm0ar = sg_req->chan_reg.dma_sm0ar; stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), dma_sm0ar); - dev_dbg(chan2dev(chan), "CT=1 <=> SM0AR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CT=1 <=> SM0AR: 0x%08x\n", stm32_dma_read(dmadev, STM32_DMA_SM0AR(id))); } else { dma_sm1ar = sg_req->chan_reg.dma_sm1ar; stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), dma_sm1ar); - dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CT=0 <=> SM1AR: 0x%08x\n", stm32_dma_read(dmadev, STM32_DMA_SM1AR(id))); } } @@ -676,7 +671,7 @@ static void stm32_dma_handle_chan_paused(struct stm32_dma_chan *chan) chan->status = DMA_PAUSED; - dev_dbg(chan2dev(chan), "vchan %p: paused\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: paused\n", &chan->vchan); } static void stm32_dma_post_resume_reconfigure(struct stm32_dma_chan *chan) @@ -728,7 +723,7 @@ static void stm32_dma_post_resume_reconfigure(struct stm32_dma_chan *chan) dma_scr |= STM32_DMA_SCR_EN; stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); - dev_dbg(chan2dev(chan), "vchan %p: reconfigured after pause/resume\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: reconfigured after pause/resume\n", &chan->vchan); } static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan, u32 scr) @@ -775,16 +770,16 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) if (sfcr & STM32_DMA_SFCR_FEIE) { if (!(scr & STM32_DMA_SCR_EN) && !(status & STM32_DMA_TCI)) - dev_err(chan2dev(chan), "FIFO Error\n"); + dev_err(vchan_chan_dev(&chan->vchan), "FIFO Error\n"); else - dev_dbg(chan2dev(chan), "FIFO over/underrun\n"); + dev_dbg(vchan_chan_dev(&chan->vchan), "FIFO over/underrun\n"); } } if (status & STM32_DMA_DMEI) { stm32_dma_irq_clear(chan, STM32_DMA_DMEI); status &= ~STM32_DMA_DMEI; if (sfcr & STM32_DMA_SCR_DMEIE) - dev_dbg(chan2dev(chan), "Direct mode overrun\n"); + dev_dbg(vchan_chan_dev(&chan->vchan), "Direct mode overrun\n"); } if (status & STM32_DMA_TCI) { @@ -803,9 +798,9 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) if (status) { stm32_dma_irq_clear(chan, status); - dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status); + dev_err(vchan_chan_dev(&chan->vchan), "DMA error: status=0x%08x\n", status); if (!(scr & STM32_DMA_SCR_EN)) - dev_err(chan2dev(chan), "chan disabled by HW\n"); + dev_err(vchan_chan_dev(&chan->vchan), "chan disabled by HW\n"); } spin_unlock(&chan->vchan.lock); @@ -820,7 +815,7 @@ static void stm32_dma_issue_pending(struct dma_chan *c) spin_lock_irqsave(&chan->vchan.lock, flags); if (vchan_issue_pending(&chan->vchan) && !chan->desc && !chan->busy) { - dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: issued\n", &chan->vchan); stm32_dma_start_transfer(chan); } @@ -922,7 +917,7 @@ static int stm32_dma_resume(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); - dev_dbg(chan2dev(chan), "vchan %p: resumed\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: resumed\n", &chan->vchan); return 0; } @@ -1059,7 +1054,7 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, break; default: - dev_err(chan2dev(chan), "Dma direction is not supported\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Dma direction is not supported\n"); return -EINVAL; } @@ -1092,12 +1087,12 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( int i, ret; if (!chan->config_init) { - dev_err(chan2dev(chan), "dma channel is not configured\n"); + dev_err(vchan_chan_dev(&chan->vchan), "dma channel is not configured\n"); return NULL; } if (sg_len < 1) { - dev_err(chan2dev(chan), "Invalid segment length %d\n", sg_len); + dev_err(vchan_chan_dev(&chan->vchan), "Invalid segment length %d\n", sg_len); return NULL; } @@ -1129,7 +1124,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( nb_data_items = desc->sg_req[i].len / buswidth; if (nb_data_items > STM32_DMA_ALIGNED_MAX_DATA_ITEMS) { - dev_err(chan2dev(chan), "nb items not supported\n"); + dev_err(vchan_chan_dev(&chan->vchan), "nb items not supported\n"); goto err; } @@ -1164,17 +1159,17 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( int i, ret; if (!buf_len || !period_len) { - dev_err(chan2dev(chan), "Invalid buffer/period len\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Invalid buffer/period len\n"); return NULL; } if (!chan->config_init) { - dev_err(chan2dev(chan), "dma channel is not configured\n"); + dev_err(vchan_chan_dev(&chan->vchan), "dma channel is not configured\n"); return NULL; } if (buf_len % period_len) { - dev_err(chan2dev(chan), "buf_len not multiple of period_len\n"); + dev_err(vchan_chan_dev(&chan->vchan), "buf_len not multiple of period_len\n"); return NULL; } @@ -1185,7 +1180,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( * terminating the DMA. */ if (chan->busy) { - dev_err(chan2dev(chan), "Request not allowed when dma busy\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Request not allowed when dma busy\n"); return NULL; } @@ -1196,7 +1191,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( nb_data_items = period_len / buswidth; if (nb_data_items > STM32_DMA_ALIGNED_MAX_DATA_ITEMS) { - dev_err(chan2dev(chan), "number of items not supported\n"); + dev_err(vchan_chan_dev(&chan->vchan), "number of items not supported\n"); return NULL; } @@ -1478,7 +1473,7 @@ static void stm32_dma_free_chan_resources(struct dma_chan *c) struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); unsigned long flags; - dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id); + dev_dbg(vchan_chan_dev(&chan->vchan), "Freeing channel %d\n", chan->id); if (chan->busy) { spin_lock_irqsave(&chan->vchan.lock, flags); @@ -1668,7 +1663,7 @@ static int stm32_dma_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma_chan_irq, 0, - dev_name(chan2dev(chan)), chan); + vchan_chan_name(&chan->vchan), chan); if (ret) { dev_err(&pdev->dev, "request_irq failed with err %d channel %d\n", diff --git a/drivers/dma/stm32/stm32-dma3.c b/drivers/dma/stm32/stm32-dma3.c index 4724e7fa0008..4fe226541efa 100644 --- a/drivers/dma/stm32/stm32-dma3.c +++ b/drivers/dma/stm32/stm32-dma3.c @@ -328,11 +328,6 @@ static inline struct stm32_dma3_swdesc *to_stm32_dma3_swdesc(struct virt_dma_des return container_of(vdesc, struct stm32_dma3_swdesc, vdesc); } -static struct device *chan2dev(struct stm32_dma3_chan *chan) -{ - return &chan->vchan.chan.dev->device; -} - static struct device *ddata2dev(struct stm32_dma3_ddata *ddata) { return ddata->dma_dev.dev; @@ -341,7 +336,7 @@ static struct device *ddata2dev(struct stm32_dma3_ddata *ddata) static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan) { struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); - struct device *dev = chan2dev(chan); + struct device *dev = vchan_chan_dev(&chan->vchan); u32 id = chan->id, offset; offset = STM32_DMA3_SECCFGR; @@ -381,21 +376,21 @@ static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan *chan, for (i = 0; i < swdesc->lli_size; i++) { hwdesc = swdesc->lli[i].hwdesc; if (i) - dev_dbg(chan2dev(chan), "V\n"); - dev_dbg(chan2dev(chan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr); - dev_dbg(chan2dev(chan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1); - dev_dbg(chan2dev(chan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2); - dev_dbg(chan2dev(chan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1); - dev_dbg(chan2dev(chan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar); - dev_dbg(chan2dev(chan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar); - dev_dbg(chan2dev(chan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr); + dev_dbg(vchan_chan_dev(&chan->vchan), "V\n"); + dev_dbg(vchan_chan_dev(&chan->vchan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr); + dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1); + dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2); + dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1); + dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar); + dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar); + dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr); } if (swdesc->cyclic) { - dev_dbg(chan2dev(chan), "|\n"); - dev_dbg(chan2dev(chan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr); + dev_dbg(vchan_chan_dev(&chan->vchan), "|\n"); + dev_dbg(vchan_chan_dev(&chan->vchan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr); } else { - dev_dbg(chan2dev(chan), "X\n"); + dev_dbg(vchan_chan_dev(&chan->vchan), "X\n"); } } @@ -411,7 +406,7 @@ static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_ch * addressed, so abort the allocation. */ if ((count * 32) > CLLR_LA) { - dev_err(chan2dev(chan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE); + dev_err(vchan_chan_dev(&chan->vchan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE); return NULL; } @@ -438,7 +433,7 @@ static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_ch return swdesc; err_pool_free: - dev_err(chan2dev(chan), "Failed to alloc descriptors\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Failed to alloc descriptors\n"); while (--i >= 0) dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr); kfree(swdesc); @@ -468,7 +463,7 @@ static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc *vdesc) static void stm32_dma3_check_user_setting(struct stm32_dma3_chan *chan) { struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); - struct device *dev = chan2dev(chan); + struct device *dev = vchan_chan_dev(&chan->vchan); u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id)); u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id)); u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id)); @@ -579,7 +574,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf u32 sap = FIELD_GET(STM32_DMA3_DT_SAP, tr_conf), sap_max_dw; u32 dap = FIELD_GET(STM32_DMA3_DT_DAP, tr_conf), dap_max_dw; - dev_dbg(chan2dev(chan), "%s from %pad to %pad\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "%s from %pad to %pad\n", dmaengine_get_direction_text(dir), &src_addr, &dst_addr); sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst); @@ -589,12 +584,12 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf /* Following conditions would raise User Setting Error interrupt */ if (!(dma_device.src_addr_widths & BIT(sdw)) || !(dma_device.dst_addr_widths & BIT(ddw))) { - dev_err(chan2dev(chan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw); + dev_err(vchan_chan_dev(&chan->vchan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw); return -EINVAL; } if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) { - dev_err(chan2dev(chan), "Only one master port, port 1 is not supported\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Only one master port, port 1 is not supported\n"); return -EINVAL; } @@ -602,7 +597,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf dap_max_dw = ddata->ports_max_dw[dap]; if ((port_is_ahb(sap_max_dw) && sdw == DMA_SLAVE_BUSWIDTH_8_BYTES) || (port_is_ahb(dap_max_dw) && ddw == DMA_SLAVE_BUSWIDTH_8_BYTES)) { - dev_err(chan2dev(chan), + dev_err(vchan_chan_dev(&chan->vchan), "8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n", sdw, ddw, sap, dap); return -EINVAL; @@ -659,7 +654,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); /* Should never reach this case as ddw is clamped down */ if (len & (ddw - 1)) { - dev_err(chan2dev(chan), + dev_err(vchan_chan_dev(&chan->vchan), "Packing mode is enabled and len is not multiple of ddw"); return -EINVAL; } @@ -695,7 +690,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); /* Should never reach this case as ddw is clamped down */ if (len & (ddw - 1)) { - dev_err(chan2dev(chan), + dev_err(vchan_chan_dev(&chan->vchan), "Packing mode is enabled and len is not multiple of ddw\n"); return -EINVAL; } @@ -740,7 +735,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); /* Should never reach this case as ddw is clamped down */ if (len & (ddw - 1)) { - dev_err(chan2dev(chan), + dev_err(vchan_chan_dev(&chan->vchan), "Packing mode is enabled and len is not multiple of ddw"); return -EINVAL; } @@ -752,7 +747,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf break; default: - dev_err(chan2dev(chan), "Direction %s not supported\n", + dev_err(vchan_chan_dev(&chan->vchan), "Direction %s not supported\n", dmaengine_get_direction_text(dir)); return -EINVAL; } @@ -761,7 +756,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf *ctr1 = _ctr1; *ctr2 = _ctr2; - dev_dbg(chan2dev(chan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n", __func__, sdw, sbl_max, ddw, dbl_max); return 0; @@ -807,7 +802,7 @@ static void stm32_dma3_chan_start(struct stm32_dma3_chan *chan) chan->dma_status = DMA_IN_PROGRESS; - dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: started\n", &chan->vchan); } static int stm32_dma3_chan_suspend(struct stm32_dma3_chan *chan, bool susp) @@ -871,7 +866,7 @@ static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan, struct dma_tx_state *txstate) { struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); - struct device *dev = chan2dev(chan); + struct device *dev = vchan_chan_dev(&chan->vchan); struct stm32_dma3_hwdesc *hwdesc; u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol; bool pack_unpack; @@ -921,7 +916,7 @@ static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan, /* Get current hwdesc and cumulate residue of pending hwdesc BNDT */ ret = stm32_dma3_chan_get_curr_hwdesc(swdesc, cllr, &residue); if (ret < 0) { - dev_err(chan2dev(chan), "Can't get residue: current hwdesc not found\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Can't get residue: current hwdesc not found\n"); return; } curr_lli = ret; @@ -957,7 +952,7 @@ static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan, skip_fifol_update: if (fifol) { - dev_dbg(chan2dev(chan), "%u byte(s) in the FIFO\n", fifol); + dev_dbg(vchan_chan_dev(&chan->vchan), "%u byte(s) in the FIFO\n", fifol); dma_set_in_flight_bytes(txstate, fifol); /* * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from @@ -987,7 +982,7 @@ static int stm32_dma3_chan_stop(struct stm32_dma3_chan *chan) /* Suspend the channel */ ret = stm32_dma3_chan_suspend(chan, true); if (ret) - dev_warn(chan2dev(chan), "%s: timeout, data might be lost\n", __func__); + dev_warn(vchan_chan_dev(&chan->vchan), "%s: timeout, data might be lost\n", __func__); } /* @@ -1034,7 +1029,7 @@ static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid) } if (csr & CSR_USEF && ccr & CCR_USEIE) { - dev_err(chan2dev(chan), "User setting error\n"); + dev_err(vchan_chan_dev(&chan->vchan), "User setting error\n"); chan->dma_status = DMA_ERROR; /* CCR.EN automatically cleared by HW */ stm32_dma3_check_user_setting(chan); @@ -1042,14 +1037,14 @@ static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid) } if (csr & CSR_ULEF && ccr & CCR_ULEIE) { - dev_err(chan2dev(chan), "Update link transfer error\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Update link transfer error\n"); chan->dma_status = DMA_ERROR; /* CCR.EN automatically cleared by HW */ stm32_dma3_chan_reset(chan); } if (csr & CSR_DTEF && ccr & CCR_DTEIE) { - dev_err(chan2dev(chan), "Data transfer error\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Data transfer error\n"); chan->dma_status = DMA_ERROR; /* CCR.EN automatically cleared by HW */ stm32_dma3_chan_reset(chan); @@ -1087,13 +1082,13 @@ static int stm32_dma3_get_chan_sem(struct stm32_dma3_chan *chan) goto bad_cid; chan->semaphore_taken = true; - dev_dbg(chan2dev(chan), "under CID1 control (semcr=0x%08x)\n", csemcr); + dev_dbg(vchan_chan_dev(&chan->vchan), "under CID1 control (semcr=0x%08x)\n", csemcr); return 0; bad_cid: chan->semaphore_taken = false; - dev_err(chan2dev(chan), "not under CID1 control (in-use by CID%d)\n", ccid); + dev_err(vchan_chan_dev(&chan->vchan), "not under CID1 control (in-use by CID%d)\n", ccid); return -EACCES; } @@ -1105,7 +1100,7 @@ static void stm32_dma3_put_chan_sem(struct stm32_dma3_chan *chan) if (chan->semaphore_taken) { writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id)); chan->semaphore_taken = false; - dev_dbg(chan2dev(chan), "no more under CID1 control\n"); + dev_dbg(vchan_chan_dev(&chan->vchan), "no more under CID1 control\n"); } } @@ -1126,11 +1121,11 @@ static int stm32_dma3_alloc_chan_resources(struct dma_chan *c) goto err_put_sync; } - chan->lli_pool = dmam_pool_create(dev_name(&c->dev->device), c->device->dev, + chan->lli_pool = dmam_pool_create(dma_chan_name(c), c->device->dev, sizeof(struct stm32_dma3_hwdesc), __alignof__(struct stm32_dma3_hwdesc), SZ_64K); if (!chan->lli_pool) { - dev_err(chan2dev(chan), "Failed to create LLI pool\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Failed to create LLI pool\n"); ret = -ENOMEM; goto err_put_sync; } @@ -1366,7 +1361,7 @@ static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan } if (count != sg_len && chan->tcem != CTR2_TCEM_CHANNEL) - dev_warn(chan2dev(chan), "Linked-list refactored, %d items instead of %d\n", + dev_warn(vchan_chan_dev(&chan->vchan), "Linked-list refactored, %d items instead of %d\n", count, sg_len); /* Enable Error interrupts */ @@ -1401,12 +1396,12 @@ static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_cha return NULL; if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) { - dev_err(chan2dev(chan), "Invalid buffer/period length\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Invalid buffer/period length\n"); return NULL; } if (buf_len % period_len) { - dev_err(chan2dev(chan), "Buffer length not multiple of period length\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Buffer length not multiple of period length\n"); return NULL; } @@ -1428,7 +1423,7 @@ static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_cha ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2, src, dst, period_len); } else { - dev_err(chan2dev(chan), "Invalid direction\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Invalid direction\n"); ret = -EINVAL; } @@ -1502,7 +1497,7 @@ static int stm32_dma3_pause(struct dma_chan *c) chan->dma_status = DMA_PAUSED; - dev_dbg(chan2dev(chan), "vchan %p: paused\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: paused\n", &chan->vchan); return 0; } @@ -1515,7 +1510,7 @@ static int stm32_dma3_resume(struct dma_chan *c) chan->dma_status = DMA_IN_PROGRESS; - dev_dbg(chan2dev(chan), "vchan %p: resumed\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: resumed\n", &chan->vchan); return 0; } @@ -1540,7 +1535,7 @@ static int stm32_dma3_terminate_all(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); vchan_dma_desc_free_list(&chan->vchan, &head); - dev_dbg(chan2dev(chan), "vchan %p: terminated\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: terminated\n", &chan->vchan); return 0; } @@ -1593,7 +1588,7 @@ static void stm32_dma3_issue_pending(struct dma_chan *c) spin_lock_irqsave(&chan->vchan.lock, flags); if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) { - dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: issued\n", &chan->vchan); stm32_dma3_chan_start(chan); } @@ -1892,10 +1887,10 @@ static int stm32_dma3_probe(struct platform_device *pdev) chan->irq = ret; ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0, - dev_name(chan2dev(chan)), chan); + vchan_chan_name(&chan->vchan), chan); if (ret) { dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n", - dev_name(chan2dev(chan))); + vchan_chan_name(&chan->vchan)); goto err_clk_disable; } } diff --git a/drivers/dma/stm32/stm32-mdma.c b/drivers/dma/stm32/stm32-mdma.c index e3bbdc9ee36e..04639e4b554e 100644 --- a/drivers/dma/stm32/stm32-mdma.c +++ b/drivers/dma/stm32/stm32-mdma.c @@ -276,11 +276,6 @@ static struct stm32_mdma_desc *to_stm32_mdma_desc(struct virt_dma_desc *vdesc) return container_of(vdesc, struct stm32_mdma_desc, vdesc); } -static struct device *chan2dev(struct stm32_mdma_chan *chan) -{ - return &chan->vchan.chan.dev->device; -} - static struct device *mdma2dev(struct stm32_mdma_device *mdma_dev) { return mdma_dev->ddev.dev; @@ -334,7 +329,7 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc( return desc; err: - dev_err(chan2dev(chan), "Failed to allocate descriptor\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Failed to allocate descriptor\n"); while (--i >= 0) dma_pool_free(chan->desc_pool, desc->node[i].hwdesc, desc->node[i].hwdesc_phys); @@ -364,7 +359,7 @@ static int stm32_mdma_get_width(struct stm32_mdma_chan *chan, case DMA_SLAVE_BUSWIDTH_8_BYTES: return ffs(width) - 1; default: - dev_err(chan2dev(chan), "Dma bus width %i not supported\n", + dev_err(vchan_chan_dev(&chan->vchan), "Dma bus width %i not supported\n", width); return -EINVAL; } @@ -422,7 +417,7 @@ static int stm32_mdma_disable_chan(struct stm32_mdma_chan *chan) dmadev->base + STM32_MDMA_CISR(id), cisr, (cisr & STM32_MDMA_CISR_CTCIF), 10, 1000); if (ret) { - dev_err(chan2dev(chan), "%s: timeout!\n", __func__); + dev_err(vchan_chan_dev(&chan->vchan), "%s: timeout!\n", __func__); return -EBUSY; } } @@ -444,7 +439,7 @@ static void stm32_mdma_stop(struct stm32_mdma_chan *chan) /* Clear interrupt status if it is there */ status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id)); if (status) { - dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "%s(): clearing interrupt: 0x%08x\n", __func__, status); stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status); } @@ -513,7 +508,7 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, /* Check burst size constraints */ if (src_maxburst * src_addr_width > STM32_MDMA_MAX_BURST || dst_maxburst * dst_addr_width > STM32_MDMA_MAX_BURST) { - dev_err(chan2dev(chan), + dev_err(vchan_chan_dev(&chan->vchan), "burst size * bus width higher than %d bytes\n", STM32_MDMA_MAX_BURST); return -EINVAL; @@ -521,7 +516,7 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, if ((!is_power_of_2(src_maxburst) && src_maxburst > 0) || (!is_power_of_2(dst_maxburst) && dst_maxburst > 0)) { - dev_err(chan2dev(chan), "burst size must be a power of 2\n"); + dev_err(vchan_chan_dev(&chan->vchan), "burst size must be a power of 2\n"); return -EINVAL; } @@ -658,7 +653,7 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, break; default: - dev_err(chan2dev(chan), "Dma direction is not supported\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Dma direction is not supported\n"); return -EINVAL; } @@ -672,16 +667,16 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan, struct stm32_mdma_desc_node *node) { - dev_dbg(chan2dev(chan), "hwdesc: %pad\n", &node->hwdesc_phys); - dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", node->hwdesc->ctcr); - dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", node->hwdesc->cbndtr); - dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", node->hwdesc->csar); - dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", node->hwdesc->cdar); - dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", node->hwdesc->cbrur); - dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", node->hwdesc->clar); - dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", node->hwdesc->ctbr); - dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", node->hwdesc->cmar); - dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n\n", node->hwdesc->cmdr); + dev_dbg(vchan_chan_dev(&chan->vchan), "hwdesc: %pad\n", &node->hwdesc_phys); + dev_dbg(vchan_chan_dev(&chan->vchan), "CTCR: 0x%08x\n", node->hwdesc->ctcr); + dev_dbg(vchan_chan_dev(&chan->vchan), "CBNDTR: 0x%08x\n", node->hwdesc->cbndtr); + dev_dbg(vchan_chan_dev(&chan->vchan), "CSAR: 0x%08x\n", node->hwdesc->csar); + dev_dbg(vchan_chan_dev(&chan->vchan), "CDAR: 0x%08x\n", node->hwdesc->cdar); + dev_dbg(vchan_chan_dev(&chan->vchan), "CBRUR: 0x%08x\n", node->hwdesc->cbrur); + dev_dbg(vchan_chan_dev(&chan->vchan), "CLAR: 0x%08x\n", node->hwdesc->clar); + dev_dbg(vchan_chan_dev(&chan->vchan), "CTBR: 0x%08x\n", node->hwdesc->ctbr); + dev_dbg(vchan_chan_dev(&chan->vchan), "CMAR: 0x%08x\n", node->hwdesc->cmar); + dev_dbg(vchan_chan_dev(&chan->vchan), "CMDR: 0x%08x\n\n", node->hwdesc->cmdr); } static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan, @@ -739,7 +734,7 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, for_each_sg(sgl, sg, sg_len, i) { if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) { - dev_err(chan2dev(chan), "Invalid block len\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Invalid block len\n"); return -EINVAL; } @@ -797,7 +792,7 @@ stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl, * for allowing another request. */ if (chan->desc && chan->desc->cyclic) { - dev_err(chan2dev(chan), + dev_err(vchan_chan_dev(&chan->vchan), "Request not allowed when dma in cyclic mode\n"); return NULL; } @@ -858,18 +853,18 @@ stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr, * for allowing another request. */ if (chan->desc && chan->desc->cyclic) { - dev_err(chan2dev(chan), + dev_err(vchan_chan_dev(&chan->vchan), "Request not allowed when dma in cyclic mode\n"); return NULL; } if (!buf_len || !period_len || period_len > STM32_MDMA_MAX_BLOCK_LEN) { - dev_err(chan2dev(chan), "Invalid buffer/period len\n"); + dev_err(vchan_chan_dev(&chan->vchan), "Invalid buffer/period len\n"); return NULL; } if (buf_len % period_len) { - dev_err(chan2dev(chan), "buf_len not multiple of period_len\n"); + dev_err(vchan_chan_dev(&chan->vchan), "buf_len not multiple of period_len\n"); return NULL; } @@ -954,7 +949,7 @@ stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src, * to allow another request */ if (chan->desc && chan->desc->cyclic) { - dev_err(chan2dev(chan), + dev_err(vchan_chan_dev(&chan->vchan), "Request not allowed when dma in cyclic mode\n"); return NULL; } @@ -1116,25 +1111,25 @@ static void stm32_mdma_dump_reg(struct stm32_mdma_chan *chan) { struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); - dev_dbg(chan2dev(chan), "CCR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CCR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id))); - dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CTCR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id))); - dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CBNDTR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id))); - dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CSAR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CSAR(chan->id))); - dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CDAR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CDAR(chan->id))); - dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CBRUR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CBRUR(chan->id))); - dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CLAR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id))); - dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CTBR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id))); - dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CMAR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CMAR(chan->id))); - dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n", + dev_dbg(vchan_chan_dev(&chan->vchan), "CMDR: 0x%08x\n", stm32_mdma_read(dmadev, STM32_MDMA_CMDR(chan->id))); } @@ -1187,7 +1182,7 @@ static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan) chan->busy = true; - dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: started\n", &chan->vchan); } static void stm32_mdma_issue_pending(struct dma_chan *c) @@ -1200,7 +1195,7 @@ static void stm32_mdma_issue_pending(struct dma_chan *c) if (!vchan_issue_pending(&chan->vchan)) goto end; - dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: issued\n", &chan->vchan); if (!chan->desc && !chan->busy) stm32_mdma_start_transfer(chan); @@ -1220,7 +1215,7 @@ static int stm32_mdma_pause(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); if (!ret) - dev_dbg(chan2dev(chan), "vchan %p: pause\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: pause\n", &chan->vchan); return ret; } @@ -1261,7 +1256,7 @@ static int stm32_mdma_resume(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); - dev_dbg(chan2dev(chan), "vchan %p: resume\n", &chan->vchan); + dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: resume\n", &chan->vchan); return 0; } @@ -1422,10 +1417,10 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) if (!(status & ien)) { spin_unlock(&chan->vchan.lock); if (chan->busy) - dev_warn(chan2dev(chan), + dev_warn(vchan_chan_dev(&chan->vchan), "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien); else - dev_dbg(chan2dev(chan), + dev_dbg(vchan_chan_dev(&chan->vchan), "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien); return IRQ_NONE; } @@ -1433,7 +1428,7 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) reg = STM32_MDMA_CIFCR(id); if (status & STM32_MDMA_CISR_TEIF) { - dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n", + dev_err(vchan_chan_dev(&chan->vchan), "Transfer Err: stat=0x%08x\n", readl_relaxed(dmadev->base + STM32_MDMA_CESR(id))); stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF); status &= ~STM32_MDMA_CISR_TEIF; @@ -1468,9 +1463,9 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) if (status) { stm32_mdma_set_bits(dmadev, reg, status); - dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status); + dev_err(vchan_chan_dev(&chan->vchan), "DMA error: status=0x%08x\n", status); if (!(ccr & STM32_MDMA_CCR_EN)) - dev_err(chan2dev(chan), "chan disabled by HW\n"); + dev_err(vchan_chan_dev(&chan->vchan), "chan disabled by HW\n"); } spin_unlock(&chan->vchan.lock); @@ -1484,13 +1479,12 @@ static int stm32_mdma_alloc_chan_resources(struct dma_chan *c) struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); int ret; - chan->desc_pool = dmam_pool_create(dev_name(&c->dev->device), - c->device->dev, + chan->desc_pool = dmam_pool_create(dma_chan_name(c), c->device->dev, sizeof(struct stm32_mdma_hwdesc), __alignof__(struct stm32_mdma_hwdesc), 0); if (!chan->desc_pool) { - dev_err(chan2dev(chan), "failed to allocate descriptor pool\n"); + dev_err(vchan_chan_dev(&chan->vchan), "failed to allocate descriptor pool\n"); return -ENOMEM; } @@ -1511,7 +1505,7 @@ static void stm32_mdma_free_chan_resources(struct dma_chan *c) struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); unsigned long flags; - dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id); + dev_dbg(vchan_chan_dev(&chan->vchan), "Freeing channel %d\n", chan->id); if (chan->busy) { spin_lock_irqsave(&chan->vchan.lock, flags); diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index d2321f7287d2..225e4a53502b 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -246,11 +246,6 @@ static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd return container_of(vd, struct sun4i_dma_contract, vd); } -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - static void set_dst_data_width_a10(u32 *p_cfg, s8 data_width) { *p_cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(data_width); @@ -428,7 +423,7 @@ static int __execute_vchan_pending(struct sun4i_dma_dev *priv, * has already submitted some work, we can't do anything else */ if (vchan->processing) { - dev_dbg(chan2dev(&vchan->vc.chan), + dev_dbg(vchan_chan_dev(&vchan->vc), "processing something to this endpoint already\n"); ret = -EBUSY; goto release_pchan; @@ -438,7 +433,7 @@ static int __execute_vchan_pending(struct sun4i_dma_dev *priv, /* Figure out which contract we're working with today */ vd = vchan_next_desc(&vchan->vc); if (!vd) { - dev_dbg(chan2dev(&vchan->vc.chan), + dev_dbg(vchan_chan_dev(&vchan->vc), "No pending contract found"); ret = 0; goto release_pchan; @@ -449,7 +444,7 @@ static int __execute_vchan_pending(struct sun4i_dma_dev *priv, /* The contract has been completed so mark it as such */ list_del(&contract->vd.node); vchan_cookie_complete(&contract->vd); - dev_dbg(chan2dev(&vchan->vc.chan), + dev_dbg(vchan_chan_dev(&vchan->vc), "Empty contract found and marked complete"); } } while (list_empty(&contract->demands)); @@ -542,7 +537,7 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, promise->cfg = SUN4I_DMA_CFG_LOADING | SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN; - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "src burst %d, dst burst %d, src buswidth %d, dst buswidth %d", sconfig->src_maxburst, sconfig->dst_maxburst, sconfig->src_addr_width, sconfig->dst_addr_width); @@ -769,7 +764,7 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, u8 ram_type, io_mode, linear_mode; if (!is_slave_direction(dir)) { - dev_err(chan2dev(chan), "Invalid DMA direction\n"); + dev_err(dmaengine_chan_dev(chan), "Invalid DMA direction\n"); return NULL; } @@ -894,7 +889,7 @@ sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return NULL; if (!is_slave_direction(dir)) { - dev_err(chan2dev(chan), "Invalid DMA direction\n"); + dev_err(dmaengine_chan_dev(chan), "Invalid DMA direction\n"); return NULL; } diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index 7704b016aed8..cf06cd9bb385 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -7,6 +7,7 @@ * Maxime Ripard */ +#include #include #include #include @@ -27,21 +28,17 @@ /* * Common registers */ -#define DMA_IRQ_EN(x) ((x) * 0x04) #define DMA_IRQ_HALF BIT(0) #define DMA_IRQ_PKG BIT(1) #define DMA_IRQ_QUEUE BIT(2) -#define DMA_IRQ_CHAN_NR 8 #define DMA_IRQ_CHAN_WIDTH 4 -#define DMA_IRQ_STAT(x) ((x) * 0x04 + 0x10) - #define DMA_STAT 0x30 /* Offset between DMA_IRQ_EN and DMA_IRQ_STAT limits number of channels */ -#define DMA_MAX_CHANNELS (DMA_IRQ_CHAN_NR * 0x10 / 4) +#define DMA_MAX_CHANNELS 16 /* * sun8i specific registers @@ -52,6 +49,20 @@ #define SUNXI_H3_SECURE_REG 0x20 #define SUNXI_H3_DMA_GATE 0x28 #define SUNXI_H3_DMA_GATE_ENABLE 0x4 + +/* + * Interrupts specific registers + */ +#define DMA_IRQ_STRIDE_A31 0x04 +#define DMA_IRQ_STRIDE_A733 0x40 +#define DMA_IRQ_EN_OFFSET_A31 0x00 +#define DMA_IRQ_EN_OFFSET_A733 0x134 +#define DMA_IRQ_STAT_OFFSET_A31 0x10 +#define DMA_IRQ_STAT_OFFSET_A733 0x138 + +#define DMA_IRQ_CHAN_NR_A31 8 +#define DMA_IRQ_CHAN_NR_A733 1 + /* * Channels specific registers */ @@ -98,8 +109,10 @@ * The LLI link physical address is also mangled, but we avoid dealing * with that by allocating LLIs from the DMA32 zone. */ -#define SRC_HIGH_ADDR(x) (((x) & 0x3U) << 16) -#define DST_HIGH_ADDR(x) (((x) & 0x3U) << 18) +#define SRC_HIGH_ADDR_MASK GENMASK(17, 16) +#define DST_HIGH_ADDR_MASK GENMASK(19, 18) +#define SRC_HIGH_ADDR_32G_MASK GENMASK(13, 11) +#define DST_HIGH_ADDR_32G_MASK GENMASK(17, 15) /* * Various hardware related defines @@ -142,8 +155,13 @@ struct sun6i_dma_config { u32 dst_burst_lengths; u32 src_addr_widths; u32 dst_addr_widths; - bool has_high_addr; + u32 src_high_addr_mask; + u32 dst_high_addr_mask; bool has_mbus_clk; + u32 irq_stride; + u32 irq_en_offset; + u32 irq_stat_offset; + u32 num_channels_per_reg; }; /* @@ -213,11 +231,6 @@ struct sun6i_dma_dev { u32 max_request; }; -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - static inline struct sun6i_dma_dev *to_sun6i_dma_dev(struct dma_device *d) { return container_of(d, struct sun6i_dma_dev, slave); @@ -234,19 +247,43 @@ to_sun6i_desc(struct dma_async_tx_descriptor *tx) return container_of(tx, struct sun6i_desc, vd.tx); } +static u32 sun6i_read_irq_en(struct sun6i_dma_dev *sdev, u32 irq_reg) +{ + return readl(sdev->base + irq_reg * sdev->cfg->irq_stride + sdev->cfg->irq_en_offset); +} + +static void sun6i_write_irq_en(struct sun6i_dma_dev *sdev, u32 irq_reg, u32 irq_val) +{ + writel(irq_val, sdev->base + irq_reg * sdev->cfg->irq_stride + sdev->cfg->irq_en_offset); +} + +static u32 sun6i_read_irq_stat(struct sun6i_dma_dev *sdev, u32 irq_reg) +{ + return readl(sdev->base + irq_reg * sdev->cfg->irq_stride + sdev->cfg->irq_stat_offset); +} + +static void sun6i_write_irq_stat(struct sun6i_dma_dev *sdev, u32 irq_reg, u32 status) +{ + writel(status, sdev->base + irq_reg * sdev->cfg->irq_stride + sdev->cfg->irq_stat_offset); +} + static inline void sun6i_dma_dump_com_regs(struct sun6i_dma_dev *sdev) { - dev_dbg(sdev->slave.dev, "Common register:\n" - "\tmask0(%04x): 0x%08x\n" - "\tmask1(%04x): 0x%08x\n" - "\tpend0(%04x): 0x%08x\n" - "\tpend1(%04x): 0x%08x\n" - "\tstats(%04x): 0x%08x\n", - DMA_IRQ_EN(0), readl(sdev->base + DMA_IRQ_EN(0)), - DMA_IRQ_EN(1), readl(sdev->base + DMA_IRQ_EN(1)), - DMA_IRQ_STAT(0), readl(sdev->base + DMA_IRQ_STAT(0)), - DMA_IRQ_STAT(1), readl(sdev->base + DMA_IRQ_STAT(1)), - DMA_STAT, readl(sdev->base + DMA_STAT)); + int i; + + for (i = 0; i < DIV_ROUND_UP(sdev->num_pchans, sdev->cfg->num_channels_per_reg); i++) { + dev_dbg(sdev->slave.dev, "Common register:\n" + "chan num %d\n" + "\tmask(%04x): 0x%08x\n" + "\tpend(%04x): 0x%08x\n" + "\tstats(%04x): 0x%08x\n", + i, + i * sdev->cfg->irq_stride + sdev->cfg->irq_en_offset, + sun6i_read_irq_en(sdev, i), + i * sdev->cfg->irq_stride + sdev->cfg->irq_stat_offset, + sun6i_read_irq_stat(sdev, i), + DMA_STAT, readl(sdev->base + DMA_STAT)); + } } static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev, @@ -399,7 +436,7 @@ static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan, struct sun6i_dma_lli *v_lli, dma_addr_t p_lli) { - dev_dbg(chan2dev(&vchan->vc.chan), + dev_dbg(vchan_chan_dev(&vchan->vc), "\n\tdesc:\tp - %pad v - 0x%p\n" "\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n" "\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n", @@ -408,16 +445,12 @@ static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan, v_lli->len, v_lli->para, v_lli->p_lli_next); } -static void sun6i_dma_free_desc(struct virt_dma_desc *vd) +static void sun6i_dma_free_desc(struct sun6i_dma_dev *sdev, + struct sun6i_desc *txd) { - struct sun6i_desc *txd = to_sun6i_desc(&vd->tx); - struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device); struct sun6i_dma_lli *v_lli, *v_next; dma_addr_t p_lli, p_next; - if (unlikely(!txd)) - return; - p_lli = txd->p_lli; v_lli = txd->v_lli; @@ -434,6 +467,17 @@ static void sun6i_dma_free_desc(struct virt_dma_desc *vd) kfree(txd); } +static void sun6i_dma_free_desc_virt(struct virt_dma_desc *vd) +{ + struct sun6i_desc *txd = to_sun6i_desc(&vd->tx); + struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device); + + if (unlikely(!txd)) + return; + + sun6i_dma_free_desc(sdev, txd); +} + static int sun6i_dma_start_desc(struct sun6i_vchan *vchan) { struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vchan->vc.chan.device); @@ -457,16 +501,16 @@ static int sun6i_dma_start_desc(struct sun6i_vchan *vchan) sun6i_dma_dump_lli(vchan, pchan->desc->v_lli, pchan->desc->p_lli); - irq_reg = pchan->idx / DMA_IRQ_CHAN_NR; - irq_offset = pchan->idx % DMA_IRQ_CHAN_NR; + irq_reg = pchan->idx / sdev->cfg->num_channels_per_reg; + irq_offset = pchan->idx % sdev->cfg->num_channels_per_reg; vchan->irq_type = vchan->cyclic ? DMA_IRQ_PKG : DMA_IRQ_QUEUE; - irq_val = readl(sdev->base + DMA_IRQ_EN(irq_reg)); + irq_val = sun6i_read_irq_en(sdev, irq_reg); irq_val &= ~((DMA_IRQ_HALF | DMA_IRQ_PKG | DMA_IRQ_QUEUE) << (irq_offset * DMA_IRQ_CHAN_WIDTH)); irq_val |= vchan->irq_type << (irq_offset * DMA_IRQ_CHAN_WIDTH); - writel(irq_val, sdev->base + DMA_IRQ_EN(irq_reg)); + sun6i_write_irq_en(sdev, irq_reg, irq_val); writel(pchan->desc->p_lli, pchan->base + DMA_CHAN_LLI_ADDR); writel(DMA_CHAN_ENABLE_START, pchan->base + DMA_CHAN_ENABLE); @@ -550,17 +594,17 @@ static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) int i, j, ret = IRQ_NONE; u32 status; - for (i = 0; i < sdev->num_pchans / DMA_IRQ_CHAN_NR; i++) { - status = readl(sdev->base + DMA_IRQ_STAT(i)); + for (i = 0; i < sdev->num_pchans / sdev->cfg->num_channels_per_reg; i++) { + status = sun6i_read_irq_stat(sdev, i); if (!status) continue; dev_dbg(sdev->slave.dev, "DMA irq status %s: 0x%x\n", str_high_low(i), status); - writel(status, sdev->base + DMA_IRQ_STAT(i)); + sun6i_write_irq_stat(sdev, i, status); - for (j = 0; (j < DMA_IRQ_CHAN_NR) && status; j++) { + for (j = 0; (j < sdev->cfg->num_channels_per_reg) && status; j++) { pchan = sdev->pchans + j; vchan = pchan->vchan; if (vchan && (status & vchan->irq_type)) { @@ -657,9 +701,10 @@ static inline void sun6i_dma_set_addr(struct sun6i_dma_dev *sdev, v_lli->src = lower_32_bits(src); v_lli->dst = lower_32_bits(dst); - if (sdev->cfg->has_high_addr) - v_lli->para |= SRC_HIGH_ADDR(upper_32_bits(src)) | - DST_HIGH_ADDR(upper_32_bits(dst)); + if (sdev->cfg->src_high_addr_mask) + v_lli->para |= + field_prep(sdev->cfg->src_high_addr_mask, upper_32_bits(src)) | + field_prep(sdev->cfg->dst_high_addr_mask, upper_32_bits(dst)); } static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( @@ -673,7 +718,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( dma_addr_t p_lli; s8 burst, width; - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n", __func__, vchan->vc.chan.chan_id, &dest, &src, len, flags); @@ -734,7 +779,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( ret = set_config(sdev, sconfig, dir, &lli_cfg); if (ret) { - dev_err(chan2dev(chan), "Invalid DMA configuration\n"); + dev_err(dmaengine_chan_dev(chan), "Invalid DMA configuration\n"); return NULL; } @@ -758,7 +803,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", __func__, vchan->vc.chan.chan_id, &sconfig->dst_addr, &sg_dma_address(sg), @@ -772,7 +817,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", __func__, vchan->vc.chan.chan_id, &sg_dma_address(sg), &sconfig->src_addr, @@ -782,7 +827,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd); } - dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli); + dev_dbg(dmaengine_chan_dev(chan), "First: %pad\n", &txd->p_lli); for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli; p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next) sun6i_dma_dump_lli(vchan, v_lli, p_lli); @@ -790,10 +835,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( return vchan_tx_prep(&vchan->vc, &txd->vd, flags); err_lli_free: - for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli; - p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next) - dma_pool_free(sdev->pool, v_lli, p_lli); - kfree(txd); + sun6i_dma_free_desc(sdev, txd); return NULL; } @@ -817,7 +859,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( ret = set_config(sdev, sconfig, dir, &lli_cfg); if (ret) { - dev_err(chan2dev(chan), "Invalid DMA configuration\n"); + dev_err(dmaengine_chan_dev(chan), "Invalid DMA configuration\n"); return NULL; } @@ -842,7 +884,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( v_lli->cfg = lli_cfg; sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n", __func__, vchan->vc.chan.chan_id, &sconfig->dst_addr, &buf_addr, @@ -854,7 +896,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( v_lli->cfg = lli_cfg; sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n", __func__, vchan->vc.chan.chan_id, &buf_addr, &sconfig->src_addr, @@ -871,10 +913,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( return vchan_tx_prep(&vchan->vc, &txd->vd, flags); err_lli_free: - for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli; - p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next) - dma_pool_free(sdev->pool, v_lli, p_lli); - kfree(txd); + sun6i_dma_free_desc(sdev, txd); return NULL; } @@ -894,7 +933,7 @@ static int sun6i_dma_pause(struct dma_chan *chan) struct sun6i_vchan *vchan = to_sun6i_vchan(chan); struct sun6i_pchan *pchan = vchan->phy; - dev_dbg(chan2dev(chan), "vchan %p: pause\n", &vchan->vc); + dev_dbg(dmaengine_chan_dev(chan), "vchan %p: pause\n", &vchan->vc); if (pchan) { writel(DMA_CHAN_PAUSE_PAUSE, @@ -915,7 +954,7 @@ static int sun6i_dma_resume(struct dma_chan *chan) struct sun6i_pchan *pchan = vchan->phy; unsigned long flags; - dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc); + dev_dbg(dmaengine_chan_dev(chan), "vchan %p: resume\n", &vchan->vc); spin_lock_irqsave(&vchan->vc.lock, flags); @@ -1024,13 +1063,13 @@ static void sun6i_dma_issue_pending(struct dma_chan *chan) if (!vchan->phy && list_empty(&vchan->node)) { list_add_tail(&vchan->node, &sdev->pending); tasklet_schedule(&sdev->task); - dev_dbg(chan2dev(chan), "vchan %p: issued\n", + dev_dbg(dmaengine_chan_dev(chan), "vchan %p: issued\n", &vchan->vc); } spin_unlock(&sdev->lock); } else { - dev_dbg(chan2dev(chan), "vchan %p: nothing to issue\n", + dev_dbg(dmaengine_chan_dev(chan), "vchan %p: nothing to issue\n", &vchan->vc); } @@ -1073,9 +1112,11 @@ static struct dma_chan *sun6i_dma_of_xlate(struct of_phandle_args *dma_spec, static inline void sun6i_kill_tasklet(struct sun6i_dma_dev *sdev) { + int i; + /* Disable all interrupts from DMA */ - writel(0, sdev->base + DMA_IRQ_EN(0)); - writel(0, sdev->base + DMA_IRQ_EN(1)); + for (i = 0; i < DMA_MAX_CHANNELS / sdev->cfg->num_channels_per_reg; i++) + sun6i_write_irq_en(sdev, i, 0); /* Prevent spurious interrupts from scheduling the tasklet */ atomic_inc(&sdev->tasklet_shutdown); @@ -1099,6 +1140,12 @@ static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev) } } +#define SUN6I_DMA_IRQ_A31_COMMON_CFG \ + .irq_stride = DMA_IRQ_STRIDE_A31, \ + .irq_en_offset = DMA_IRQ_EN_OFFSET_A31, \ + .irq_stat_offset = DMA_IRQ_STAT_OFFSET_A31, \ + .num_channels_per_reg = DMA_IRQ_CHAN_NR_A31, + /* * For A31: * @@ -1130,6 +1177,7 @@ static struct sun6i_dma_config sun6i_a31_dma_cfg = { .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), + SUN6I_DMA_IRQ_A31_COMMON_CFG }; /* @@ -1153,6 +1201,7 @@ static struct sun6i_dma_config sun8i_a23_dma_cfg = { .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), + SUN6I_DMA_IRQ_A31_COMMON_CFG }; static struct sun6i_dma_config sun8i_a83t_dma_cfg = { @@ -1171,6 +1220,7 @@ static struct sun6i_dma_config sun8i_a83t_dma_cfg = { .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), + SUN6I_DMA_IRQ_A31_COMMON_CFG }; /* @@ -1198,6 +1248,7 @@ static struct sun6i_dma_config sun8i_h3_dma_cfg = { BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), + SUN6I_DMA_IRQ_A31_COMMON_CFG }; /* @@ -1219,6 +1270,7 @@ static struct sun6i_dma_config sun50i_a64_dma_cfg = { BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), + SUN6I_DMA_IRQ_A31_COMMON_CFG }; /* @@ -1240,8 +1292,10 @@ static struct sun6i_dma_config sun50i_a100_dma_cfg = { BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), - .has_high_addr = true, + .src_high_addr_mask = SRC_HIGH_ADDR_MASK, + .dst_high_addr_mask = DST_HIGH_ADDR_MASK, .has_mbus_clk = true, + SUN6I_DMA_IRQ_A31_COMMON_CFG }; /* @@ -1264,6 +1318,33 @@ static struct sun6i_dma_config sun50i_h6_dma_cfg = { BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), .has_mbus_clk = true, + SUN6I_DMA_IRQ_A31_COMMON_CFG +}; + +/* + * The A733 binding uses the number of dma channels from the + * device tree node. + */ +static struct sun6i_dma_config sun60i_a733_dma_cfg = { + .clock_autogate_enable = sun6i_enable_clock_autogate_h3, + .set_burst_length = sun6i_set_burst_length_h3, + .set_drq = sun6i_set_drq_h6, + .set_mode = sun6i_set_mode_h6, + .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), + .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), + .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), + .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), + .src_high_addr_mask = SRC_HIGH_ADDR_32G_MASK, + .dst_high_addr_mask = DST_HIGH_ADDR_32G_MASK, + .has_mbus_clk = true, + .irq_stride = DMA_IRQ_STRIDE_A733, + .irq_en_offset = DMA_IRQ_EN_OFFSET_A733, + .irq_stat_offset = DMA_IRQ_STAT_OFFSET_A733, + .num_channels_per_reg = DMA_IRQ_CHAN_NR_A733, }; /* @@ -1287,6 +1368,7 @@ static struct sun6i_dma_config sun8i_v3s_dma_cfg = { .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), + SUN6I_DMA_IRQ_A31_COMMON_CFG }; static const struct of_device_id sun6i_dma_match[] = { @@ -1299,6 +1381,7 @@ static const struct of_device_id sun6i_dma_match[] = { { .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg }, { .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg }, { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg }, + { .compatible = "allwinner,sun60i-a733-dma", .data = &sun60i_a733_dma_cfg }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, sun6i_dma_match); @@ -1429,7 +1512,7 @@ static int sun6i_dma_probe(struct platform_device *pdev) struct sun6i_vchan *vchan = &sdc->vchans[i]; INIT_LIST_HEAD(&vchan->node); - vchan->vc.desc_free = sun6i_dma_free_desc; + vchan->vc.desc_free = sun6i_dma_free_desc_virt; vchan_init(&vchan->vc, &sdc->slave); } diff --git a/drivers/dma/switchtec_dma.c b/drivers/dma/switchtec_dma.c index c8e2169877e9..a386c3a284e6 100644 --- a/drivers/dma/switchtec_dma.c +++ b/drivers/dma/switchtec_dma.c @@ -406,7 +406,7 @@ static int disable_channel(struct switchtec_dma_chan *swdma_chan) static void switchtec_dma_cleanup_completed(struct switchtec_dma_chan *swdma_chan) { - struct device *chan_dev = &swdma_chan->dma_chan.dev->device; + struct device *chan_dev = dmaengine_chan_dev(&swdma_chan->dma_chan); struct switchtec_dma_desc *desc; struct switchtec_dma_hw_ce *ce; struct dmaengine_result res; @@ -851,7 +851,7 @@ static irqreturn_t switchtec_dma_chan_status_isr(int irq, void *dma) list_for_each_entry(chan, &dma_dev->channels, device_node) { swdma_chan = container_of(chan, struct switchtec_dma_chan, dma_chan); - chan_dev = &swdma_chan->dma_chan.dev->device; + chan_dev = dmaengine_chan_dev(&swdma_chan->dma_chan); chan_hw = swdma_chan->mmio_chan_hw; rcu_read_lock(); @@ -886,14 +886,18 @@ static void switchtec_dma_free_desc(struct switchtec_dma_chan *swdma_chan) if (swdma_chan->hw_sq) dma_free_coherent(swdma_dev->dma_dev.dev, size, swdma_chan->hw_sq, swdma_chan->dma_addr_sq); + swdma_chan->hw_sq = NULL; size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq); if (swdma_chan->hw_cq) dma_free_coherent(swdma_dev->dma_dev.dev, size, swdma_chan->hw_cq, swdma_chan->dma_addr_cq); + swdma_chan->hw_cq = NULL; - for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) + for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) { kfree(swdma_chan->desc_ring[i]); + swdma_chan->desc_ring[i] = NULL; + } } static int switchtec_dma_alloc_desc(struct switchtec_dma_chan *swdma_chan) @@ -984,15 +988,15 @@ static int switchtec_dma_alloc_chan_resources(struct dma_chan *chan) rc = enable_channel(swdma_chan); if (rc) - return rc; + goto err_free_desc; rc = reset_channel(swdma_chan); if (rc) - return rc; + goto err_disable_channel; rc = unhalt_channel(swdma_chan); if (rc) - return rc; + goto err_disable_channel; swdma_chan->ring_active = true; swdma_chan->comp_ring_active = true; @@ -1003,28 +1007,43 @@ static int switchtec_dma_alloc_chan_resources(struct dma_chan *chan) rcu_read_lock(); if (!rcu_dereference(swdma_dev->pdev)) { rcu_read_unlock(); - return -ENODEV; + rc = -ENODEV; + goto err_ring_inactive; } perf_cfg = readl(&swdma_chan->mmio_chan_fw->perf_cfg); rcu_read_unlock(); - dev_dbg(&chan->dev->device, "Burst Size: 0x%x\n", + dev_dbg(dmaengine_chan_dev(chan), "Burst Size: 0x%x\n", FIELD_GET(PERF_BURST_SIZE_MASK, perf_cfg)); - dev_dbg(&chan->dev->device, "Burst Scale: 0x%x\n", + dev_dbg(dmaengine_chan_dev(chan), "Burst Scale: 0x%x\n", FIELD_GET(PERF_BURST_SCALE_MASK, perf_cfg)); - dev_dbg(&chan->dev->device, "Interval: 0x%x\n", + dev_dbg(dmaengine_chan_dev(chan), "Interval: 0x%x\n", FIELD_GET(PERF_INTERVAL_MASK, perf_cfg)); - dev_dbg(&chan->dev->device, "Arb Weight: 0x%x\n", + dev_dbg(dmaengine_chan_dev(chan), "Arb Weight: 0x%x\n", FIELD_GET(PERF_ARB_WEIGHT_MASK, perf_cfg)); - dev_dbg(&chan->dev->device, "MRRS: 0x%x\n", + dev_dbg(dmaengine_chan_dev(chan), "MRRS: 0x%x\n", FIELD_GET(PERF_MRRS_MASK, perf_cfg)); return SWITCHTEC_DMA_SQ_SIZE; + +err_ring_inactive: + spin_lock_bh(&swdma_chan->submit_lock); + swdma_chan->ring_active = false; + spin_unlock_bh(&swdma_chan->submit_lock); + + spin_lock_bh(&swdma_chan->complete_lock); + swdma_chan->comp_ring_active = false; + spin_unlock_bh(&swdma_chan->complete_lock); +err_disable_channel: + disable_channel(swdma_chan); +err_free_desc: + switchtec_dma_free_desc(swdma_chan); + return rc; } static void switchtec_dma_free_chan_resources(struct dma_chan *chan) @@ -1145,15 +1164,34 @@ static int switchtec_dma_chan_free(struct pci_dev *pdev, return 0; } -static int switchtec_dma_chans_release(struct pci_dev *pdev, - struct switchtec_dma_dev *swdma_dev) +static void switchtec_dma_chans_release(struct pci_dev *pdev, + struct switchtec_dma_dev *swdma_dev) { int i; for (i = 0; i < swdma_dev->chan_cnt; i++) switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]); +} - return 0; +static void switchtec_dma_chans_free(struct switchtec_dma_dev *swdma_dev) +{ + int i; + + for (i = 0; i < swdma_dev->chan_cnt; i++) { + list_del(&swdma_dev->swdma_chans[i]->dma_chan.device_node); + kfree(swdma_dev->swdma_chans[i]); + } + + kfree(swdma_dev->swdma_chans); +} + +static void switchtec_dma_chans_disable(struct pci_dev *pdev, + struct switchtec_dma_dev *swdma_dev) +{ + if (swdma_dev->chan_status_irq >= 0) { + pci_free_irq(pdev, swdma_dev->chan_status_irq, swdma_dev); + swdma_dev->chan_status_irq = -1; + } } static int switchtec_dma_chans_enumerate(struct switchtec_dma_dev *swdma_dev, @@ -1180,7 +1218,7 @@ static int switchtec_dma_chans_enumerate(struct switchtec_dma_dev *swdma_dev, if (rc) { dev_err(&pdev->dev, "Channel %d: init channel failed\n", i); - chan_cnt = i; + swdma_dev->chan_cnt = i; goto err_exit; } } @@ -1188,10 +1226,9 @@ static int switchtec_dma_chans_enumerate(struct switchtec_dma_dev *swdma_dev, return chan_cnt; err_exit: - for (i = 0; i < chan_cnt; i++) - switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]); - - kfree(swdma_dev->swdma_chans); + switchtec_dma_chans_disable(pdev, swdma_dev); + switchtec_dma_chans_release(pdev, swdma_dev); + switchtec_dma_chans_free(swdma_dev); return rc; } @@ -1200,12 +1237,8 @@ static void switchtec_dma_release(struct dma_device *dma_dev) { struct switchtec_dma_dev *swdma_dev = container_of(dma_dev, struct switchtec_dma_dev, dma_dev); - int i; - for (i = 0; i < swdma_dev->chan_cnt; i++) - kfree(swdma_dev->swdma_chans[i]); - - kfree(swdma_dev->swdma_chans); + switchtec_dma_chans_free(swdma_dev); put_device(dma_dev->dev); kfree(swdma_dev); @@ -1225,6 +1258,8 @@ static int switchtec_dma_create(struct pci_dev *pdev) if (!swdma_dev) return -ENOMEM; + swdma_dev->chan_status_irq = -1; + swdma_dev->bar = ioremap(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); @@ -1295,11 +1330,13 @@ static int switchtec_dma_create(struct pci_dev *pdev) return 0; err_chans_release_exit: + switchtec_dma_chans_disable(pdev, swdma_dev); switchtec_dma_chans_release(pdev, swdma_dev); + switchtec_dma_chans_free(swdma_dev); err_exit: - if (swdma_dev->chan_status_irq) - free_irq(swdma_dev->chan_status_irq, swdma_dev); + if (swdma_dev->chan_status_irq >= 0) + pci_free_irq(pdev, swdma_dev->chan_status_irq, swdma_dev); iounmap(swdma_dev->bar); kfree(swdma_dev); @@ -1342,6 +1379,7 @@ static int switchtec_dma_probe(struct pci_dev *pdev, static void switchtec_dma_remove(struct pci_dev *pdev) { struct switchtec_dma_dev *swdma_dev = pci_get_drvdata(pdev); + void __iomem *bar = swdma_dev->bar; switchtec_dma_chans_release(pdev, swdma_dev); @@ -1354,7 +1392,7 @@ static void switchtec_dma_remove(struct pci_dev *pdev) dma_async_device_unregister(&swdma_dev->dma_dev); - iounmap(swdma_dev->bar); + iounmap(bar); pci_release_mem_regions(pdev); pci_disable_device(pdev); } diff --git a/drivers/dma/tegra186-gpc-dma.c b/drivers/dma/tegra186-gpc-dma.c index 64cedef1050a..e76c58f213d8 100644 --- a/drivers/dma/tegra186-gpc-dma.c +++ b/drivers/dma/tegra186-gpc-dma.c @@ -1537,7 +1537,7 @@ static int tegra_dma_probe(struct platform_device *pdev) * the channels available and registered for the DMA device are used. */ list_for_each_entry(chan, &tdma->dma_dev.channels, device_node) { - chdev = &chan->dev->device; + chdev = dmaengine_chan_dev(chan); tdc = to_tegra_dma_chan(chan); if (use_iommu_map) { @@ -1554,7 +1554,7 @@ static int tegra_dma_probe(struct platform_device *pdev) return dev_err_probe(chdev, -EINVAL, "Failed to get stream ID for channel %d\n", tdc->id); - chan->dev->chan_dma_dev = true; + chan->chan_dev->chan_dma_dev = true; } /* program stream-id for this channel */ diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index 640b8a218c9a..7ff8c0241f61 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -256,7 +256,7 @@ txd_to_tegra_dma_desc(struct dma_async_tx_descriptor *td) static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) { - return &tdc->dma_chan.dev->device; + return dmaengine_chan_dev(&tdc->dma_chan); } static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx); diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index fb21e0df5ab7..78a67cb9d6e0 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -422,16 +422,16 @@ static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel) { - struct device *chan_dev = &chan->dev->device; + struct device *chan_dev = dmaengine_chan_dev(chan); if (asel == 0) { /* No special handling for the channel */ - chan->dev->chan_dma_dev = false; + chan->chan_dev->chan_dma_dev = false; dev_clear_dma_coherent(chan_dev); chan_dev->dma_parms = NULL; } else if (asel == 14 || asel == 15) { - chan->dev->chan_dma_dev = true; + chan->chan_dev->chan_dma_dev = true; dev_set_dma_coherent(chan_dev); dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48)); diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c index 4fc5119854ec..0acd7a36a3d0 100644 --- a/drivers/dma/timb_dma.c +++ b/drivers/dma/timb_dma.c @@ -90,14 +90,9 @@ struct timb_dma { struct tasklet_struct tasklet; struct timb_dma_chan channels[]; }; - -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} static struct device *chan2dmadev(struct dma_chan *chan) { - return chan2dev(chan)->parent->parent; + return dmaengine_chan_dev(chan)->parent->parent; } static struct timb_dma *tdchantotd(struct timb_dma_chan *td_chan) @@ -117,7 +112,7 @@ static void __td_enable_chan_irq(struct timb_dma_chan *td_chan) /* enable interrupt for this channel */ ier = ioread32(td->membase + TIMBDMA_IER); ier |= 1 << id; - dev_dbg(chan2dev(&td_chan->chan), "Enabling irq: %d, IER: 0x%x\n", id, + dev_dbg(dmaengine_chan_dev(&td_chan->chan), "Enabling irq: %d, IER: 0x%x\n", id, ier); iowrite32(ier, td->membase + TIMBDMA_IER); } @@ -131,7 +126,7 @@ static bool __td_dma_done_ack(struct timb_dma_chan *td_chan) u32 isr; bool done = false; - dev_dbg(chan2dev(&td_chan->chan), "Checking irq: %d, td: %p\n", id, td); + dev_dbg(dmaengine_chan_dev(&td_chan->chan), "Checking irq: %d, td: %p\n", id, td); isr = ioread32(td->membase + TIMBDMA_ISR) & (1 << id); if (isr) { @@ -146,18 +141,18 @@ static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc, struct scatterlist *sg, bool last) { if (sg_dma_len(sg) > USHRT_MAX) { - dev_err(chan2dev(&td_chan->chan), "Too big sg element\n"); + dev_err(dmaengine_chan_dev(&td_chan->chan), "Too big sg element\n"); return -EINVAL; } /* length must be word aligned */ if (sg_dma_len(sg) % sizeof(u32)) { - dev_err(chan2dev(&td_chan->chan), "Incorrect length: %d\n", + dev_err(dmaengine_chan_dev(&td_chan->chan), "Incorrect length: %d\n", sg_dma_len(sg)); return -EINVAL; } - dev_dbg(chan2dev(&td_chan->chan), "desc: %p, addr: 0x%llx\n", + dev_dbg(dmaengine_chan_dev(&td_chan->chan), "desc: %p, addr: 0x%llx\n", dma_desc, (unsigned long long)sg_dma_address(sg)); dma_desc[7] = (sg_dma_address(sg) >> 24) & 0xff; @@ -180,7 +175,7 @@ static void __td_start_dma(struct timb_dma_chan *td_chan) struct timb_dma_desc *td_desc; if (td_chan->ongoing) { - dev_err(chan2dev(&td_chan->chan), + dev_err(dmaengine_chan_dev(&td_chan->chan), "Transfer already ongoing\n"); return; } @@ -188,7 +183,7 @@ static void __td_start_dma(struct timb_dma_chan *td_chan) td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc, desc_node); - dev_dbg(chan2dev(&td_chan->chan), + dev_dbg(dmaengine_chan_dev(&td_chan->chan), "td_chan: %p, chan: %d, membase: %p\n", td_chan, td_chan->chan.chan_id, td_chan->membase); @@ -230,7 +225,7 @@ static void __td_finish(struct timb_dma_chan *td_chan) desc_node); txd = &td_desc->txd; - dev_dbg(chan2dev(&td_chan->chan), "descriptor %u complete\n", + dev_dbg(dmaengine_chan_dev(&td_chan->chan), "descriptor %u complete\n", txd->cookie); /* make sure to stop the transfer */ @@ -284,7 +279,7 @@ static void __td_start_next(struct timb_dma_chan *td_chan) td_desc = list_entry(td_chan->queue.next, struct timb_dma_desc, desc_node); - dev_dbg(chan2dev(&td_chan->chan), "%s: started %u\n", + dev_dbg(dmaengine_chan_dev(&td_chan->chan), "%s: started %u\n", __func__, td_desc->txd.cookie); list_move(&td_desc->desc_node, &td_chan->active_list); @@ -303,12 +298,12 @@ static dma_cookie_t td_tx_submit(struct dma_async_tx_descriptor *txd) cookie = dma_cookie_assign(txd); if (list_empty(&td_chan->active_list)) { - dev_dbg(chan2dev(txd->chan), "%s: started %u\n", __func__, + dev_dbg(dmaengine_chan_dev(txd->chan), "%s: started %u\n", __func__, txd->cookie); list_add_tail(&td_desc->desc_node, &td_chan->active_list); __td_start_dma(td_chan); } else { - dev_dbg(chan2dev(txd->chan), "tx_submit: queued %u\n", + dev_dbg(dmaengine_chan_dev(txd->chan), "tx_submit: queued %u\n", txd->cookie); list_add_tail(&td_desc->desc_node, &td_chan->queue); @@ -344,7 +339,7 @@ static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan) err = dma_mapping_error(chan2dmadev(chan), td_desc->txd.phys); if (err) { - dev_err(chan2dev(chan), "DMA mapping error: %d\n", err); + dev_err(dmaengine_chan_dev(chan), "DMA mapping error: %d\n", err); goto err; } @@ -359,7 +354,7 @@ static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan) static void td_free_desc(struct timb_dma_desc *td_desc) { - dev_dbg(chan2dev(td_desc->txd.chan), "Freeing desc: %p\n", td_desc); + dev_dbg(dmaengine_chan_dev(td_desc->txd.chan), "Freeing desc: %p\n", td_desc); dma_unmap_single(chan2dmadev(td_desc->txd.chan), td_desc->txd.phys, td_desc->desc_list_len, DMA_TO_DEVICE); @@ -370,7 +365,7 @@ static void td_free_desc(struct timb_dma_desc *td_desc) static void td_desc_put(struct timb_dma_chan *td_chan, struct timb_dma_desc *td_desc) { - dev_dbg(chan2dev(&td_chan->chan), "Putting desc: %p\n", td_desc); + dev_dbg(dmaengine_chan_dev(&td_chan->chan), "Putting desc: %p\n", td_desc); spin_lock_bh(&td_chan->lock); list_add(&td_desc->desc_node, &td_chan->free_list); @@ -390,7 +385,7 @@ static struct timb_dma_desc *td_desc_get(struct timb_dma_chan *td_chan) ret = td_desc; break; } - dev_dbg(chan2dev(&td_chan->chan), "desc %p not ACKed\n", + dev_dbg(dmaengine_chan_dev(&td_chan->chan), "desc %p not ACKed\n", td_desc); } spin_unlock_bh(&td_chan->lock); @@ -404,7 +399,7 @@ static int td_alloc_chan_resources(struct dma_chan *chan) container_of(chan, struct timb_dma_chan, chan); int i; - dev_dbg(chan2dev(chan), "%s: entry\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: entry\n", __func__); BUG_ON(!list_empty(&td_chan->free_list)); for (i = 0; i < td_chan->descs; i++) { @@ -413,7 +408,7 @@ static int td_alloc_chan_resources(struct dma_chan *chan) if (i) break; else { - dev_err(chan2dev(chan), + dev_err(dmaengine_chan_dev(chan), "Couldn't allocate any descriptors\n"); return -ENOMEM; } @@ -436,7 +431,7 @@ static void td_free_chan_resources(struct dma_chan *chan) struct timb_dma_desc *td_desc, *_td_desc; LIST_HEAD(list); - dev_dbg(chan2dev(chan), "%s: Entry\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: Entry\n", __func__); /* check that all descriptors are free */ BUG_ON(!list_empty(&td_chan->active_list)); @@ -447,7 +442,7 @@ static void td_free_chan_resources(struct dma_chan *chan) spin_unlock_bh(&td_chan->lock); list_for_each_entry_safe(td_desc, _td_desc, &list, desc_node) { - dev_dbg(chan2dev(chan), "%s: Freeing desc: %p\n", __func__, + dev_dbg(dmaengine_chan_dev(chan), "%s: Freeing desc: %p\n", __func__, td_desc); td_free_desc(td_desc); } @@ -458,11 +453,11 @@ static enum dma_status td_tx_status(struct dma_chan *chan, dma_cookie_t cookie, { enum dma_status ret; - dev_dbg(chan2dev(chan), "%s: Entry\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: Entry\n", __func__); ret = dma_cookie_status(chan, cookie, txstate); - dev_dbg(chan2dev(chan), "%s: exit, ret: %d\n", __func__, ret); + dev_dbg(dmaengine_chan_dev(chan), "%s: exit, ret: %d\n", __func__, ret); return ret; } @@ -472,7 +467,7 @@ static void td_issue_pending(struct dma_chan *chan) struct timb_dma_chan *td_chan = container_of(chan, struct timb_dma_chan, chan); - dev_dbg(chan2dev(chan), "%s: Entry\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: Entry\n", __func__); spin_lock_bh(&td_chan->lock); if (!list_empty(&td_chan->active_list)) @@ -499,20 +494,20 @@ static struct dma_async_tx_descriptor *td_prep_slave_sg(struct dma_chan *chan, unsigned int desc_usage = 0; if (!sgl || !sg_len) { - dev_err(chan2dev(chan), "%s: No SG list\n", __func__); + dev_err(dmaengine_chan_dev(chan), "%s: No SG list\n", __func__); return NULL; } /* even channels are for RX, odd for TX */ if (td_chan->direction != direction) { - dev_err(chan2dev(chan), + dev_err(dmaengine_chan_dev(chan), "Requesting channel in wrong direction\n"); return NULL; } td_desc = td_desc_get(td_chan); if (!td_desc) { - dev_err(chan2dev(chan), "Not enough descriptors available\n"); + dev_err(dmaengine_chan_dev(chan), "Not enough descriptors available\n"); return NULL; } @@ -521,14 +516,14 @@ static struct dma_async_tx_descriptor *td_prep_slave_sg(struct dma_chan *chan, for_each_sg(sgl, sg, sg_len, i) { int err; if (desc_usage > td_desc->desc_list_len) { - dev_err(chan2dev(chan), "No descriptor space\n"); + dev_err(dmaengine_chan_dev(chan), "No descriptor space\n"); return NULL; } err = td_fill_desc(td_chan, td_desc->desc_list + desc_usage, sg, i == (sg_len - 1)); if (err) { - dev_err(chan2dev(chan), "Failed to update desc: %d\n", + dev_err(dmaengine_chan_dev(chan), "Failed to update desc: %d\n", err); td_desc_put(td_chan, td_desc); return NULL; @@ -548,7 +543,7 @@ static int td_terminate_all(struct dma_chan *chan) container_of(chan, struct timb_dma_chan, chan); struct timb_dma_desc *td_desc, *_td_desc; - dev_dbg(chan2dev(chan), "%s: Entry\n", __func__); + dev_dbg(dmaengine_chan_dev(chan), "%s: Entry\n", __func__); /* first the easy part, put the queue into the free list */ spin_lock_bh(&td_chan->lock); diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c index 05622b68a936..ecfaa6a76516 100644 --- a/drivers/dma/txx9dmac.c +++ b/drivers/dma/txx9dmac.c @@ -125,14 +125,9 @@ static struct txx9dmac_regs32 __iomem *__txx9dmac_regs32( #define dma_writel(ddev, name, val) \ (__is_dmac64(ddev) ? \ dma64_writel(ddev, name, val) : dma32_writel(ddev, name, val)) - -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} static struct device *chan2parent(struct dma_chan *chan) { - return chan->dev->device.parent; + return dmaengine_chan_dev(chan)->parent; } static struct txx9dmac_desc * @@ -218,12 +213,12 @@ static struct txx9dmac_desc *txx9dmac_desc_get(struct txx9dmac_chan *dc) ret = desc; break; } - dev_dbg(chan2dev(&dc->chan), "desc %p not ACKed\n", desc); + dev_dbg(dmaengine_chan_dev(&dc->chan), "desc %p not ACKed\n", desc); i++; } spin_unlock_bh(&dc->lock); - dev_vdbg(chan2dev(&dc->chan), "scanned %u descriptors on freelist\n", + dev_vdbg(dmaengine_chan_dev(&dc->chan), "scanned %u descriptors on freelist\n", i); if (!ret) { ret = txx9dmac_desc_alloc(dc, GFP_ATOMIC); @@ -232,7 +227,7 @@ static struct txx9dmac_desc *txx9dmac_desc_get(struct txx9dmac_chan *dc) dc->descs_allocated++; spin_unlock_bh(&dc->lock); } else - dev_err(chan2dev(&dc->chan), + dev_err(dmaengine_chan_dev(&dc->chan), "not enough descriptors available\n"); } return ret; @@ -267,11 +262,11 @@ static void txx9dmac_desc_put(struct txx9dmac_chan *dc, spin_lock_bh(&dc->lock); list_for_each_entry(child, &desc->tx_list, desc_node) - dev_vdbg(chan2dev(&dc->chan), + dev_vdbg(dmaengine_chan_dev(&dc->chan), "moving child desc %p to freelist\n", child); list_splice_init(&desc->tx_list, &dc->free_list); - dev_vdbg(chan2dev(&dc->chan), "moving desc %p to freelist\n", + dev_vdbg(dmaengine_chan_dev(&dc->chan), "moving desc %p to freelist\n", desc); list_add(&desc->desc_node, &dc->free_list); spin_unlock_bh(&dc->lock); @@ -283,7 +278,7 @@ static void txx9dmac_desc_put(struct txx9dmac_chan *dc, static void txx9dmac_dump_regs(struct txx9dmac_chan *dc) { if (is_dmac64(dc)) - dev_err(chan2dev(&dc->chan), + dev_err(dmaengine_chan_dev(&dc->chan), " CHAR: %#llx SAR: %#llx DAR: %#llx CNTR: %#x" " SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n", (u64)channel64_read_CHAR(dc), @@ -295,7 +290,7 @@ static void txx9dmac_dump_regs(struct txx9dmac_chan *dc) channel64_readl(dc, CCR), channel64_readl(dc, CSR)); else - dev_err(chan2dev(&dc->chan), + dev_err(dmaengine_chan_dev(&dc->chan), " CHAR: %#x SAR: %#x DAR: %#x CNTR: %#x" " SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n", channel32_readl(dc, CHAR), @@ -333,11 +328,11 @@ static void txx9dmac_dostart(struct txx9dmac_chan *dc, struct txx9dmac_slave *ds = dc->chan.private; u32 sai, dai; - dev_vdbg(chan2dev(&dc->chan), "dostart %u %p\n", + dev_vdbg(dmaengine_chan_dev(&dc->chan), "dostart %u %p\n", first->txd.cookie, first); /* ASSERT: channel is idle */ if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) { - dev_err(chan2dev(&dc->chan), + dev_err(dmaengine_chan_dev(&dc->chan), "BUG: Attempted to start non-idle channel\n"); txx9dmac_dump_regs(dc); /* The tasklet will hopefully advance the queue... */ @@ -402,7 +397,7 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc, struct dmaengine_desc_callback cb; struct dma_async_tx_descriptor *txd = &desc->txd; - dev_vdbg(chan2dev(&dc->chan), "descriptor %u %p complete\n", + dev_vdbg(dmaengine_chan_dev(&dc->chan), "descriptor %u %p complete\n", txd->cookie, desc); dma_cookie_complete(txd); @@ -469,11 +464,11 @@ static void txx9dmac_dump_desc(struct txx9dmac_chan *dc, { if (is_dmac64(dc)) { #ifdef TXX9_DMA_USE_SIMPLE_CHAIN - dev_crit(chan2dev(&dc->chan), + dev_crit(dmaengine_chan_dev(&dc->chan), " desc: ch%#llx s%#llx d%#llx c%#x\n", (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR); #else - dev_crit(chan2dev(&dc->chan), + dev_crit(dmaengine_chan_dev(&dc->chan), " desc: ch%#llx s%#llx d%#llx c%#x" " si%#x di%#x cc%#x cs%#x\n", (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR, @@ -482,11 +477,11 @@ static void txx9dmac_dump_desc(struct txx9dmac_chan *dc, } else { struct txx9dmac_hwdesc32 *d = (struct txx9dmac_hwdesc32 *)desc; #ifdef TXX9_DMA_USE_SIMPLE_CHAIN - dev_crit(chan2dev(&dc->chan), + dev_crit(dmaengine_chan_dev(&dc->chan), " desc: ch%#x s%#x d%#x c%#x\n", d->CHAR, d->SAR, d->DAR, d->CNTR); #else - dev_crit(chan2dev(&dc->chan), + dev_crit(dmaengine_chan_dev(&dc->chan), " desc: ch%#x s%#x d%#x c%#x" " si%#x di%#x cc%#x cs%#x\n", d->CHAR, d->SAR, d->DAR, d->CNTR, @@ -506,7 +501,7 @@ static void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr) * borked. Since we don't have any way to report errors, we'll * just have to scream loudly and try to carry on. */ - dev_crit(chan2dev(&dc->chan), "Abnormal Chain Completion\n"); + dev_crit(dmaengine_chan_dev(&dc->chan), "Abnormal Chain Completion\n"); txx9dmac_dump_regs(dc); bad_desc = txx9dmac_first_active(dc); @@ -523,7 +518,7 @@ static void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr) if (!list_empty(&dc->active_list)) txx9dmac_dostart(dc, txx9dmac_first_active(dc)); - dev_crit(chan2dev(&dc->chan), + dev_crit(dmaengine_chan_dev(&dc->chan), "Bad descriptor submitted for DMA! (cookie: %d)\n", bad_desc->txd.cookie); txx9dmac_dump_desc(dc, &bad_desc->hwdesc); @@ -558,7 +553,7 @@ static void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc) if (!(csr & TXX9_DMA_CSR_CHNEN)) chain = 0; /* last descriptor of this chain */ - dev_vdbg(chan2dev(&dc->chan), "scan_descriptors: char=%#llx\n", + dev_vdbg(dmaengine_chan_dev(&dc->chan), "scan_descriptors: char=%#llx\n", (u64)chain); list_for_each_entry_safe(desc, _desc, &dc->active_list, desc_node) { @@ -589,7 +584,7 @@ static void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc) return; } - dev_err(chan2dev(&dc->chan), + dev_err(dmaengine_chan_dev(&dc->chan), "BUG: All descriptors done, but channel not idle!\n"); /* Try to continue after resetting the channel... */ @@ -609,7 +604,7 @@ static void txx9dmac_chan_tasklet(struct tasklet_struct *t) dc = from_tasklet(dc, t, tasklet); csr = channel_readl(dc, CSR); - dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n", csr); + dev_vdbg(dmaengine_chan_dev(&dc->chan), "tasklet: status=%x\n", csr); spin_lock(&dc->lock); if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC | @@ -625,7 +620,7 @@ static irqreturn_t txx9dmac_chan_interrupt(int irq, void *dev_id) { struct txx9dmac_chan *dc = dev_id; - dev_vdbg(chan2dev(&dc->chan), "interrupt: status=%#x\n", + dev_vdbg(dmaengine_chan_dev(&dc->chan), "interrupt: status=%#x\n", channel_readl(dc, CSR)); tasklet_schedule(&dc->tasklet); @@ -654,7 +649,7 @@ static void txx9dmac_tasklet(struct tasklet_struct *t) if ((mcr >> (24 + i)) & 0x11) { dc = ddev->chan[i]; csr = channel_readl(dc, CSR); - dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n", + dev_vdbg(dmaengine_chan_dev(&dc->chan), "tasklet: status=%x\n", csr); spin_lock(&dc->lock); if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC | @@ -696,7 +691,7 @@ static dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx) spin_lock_bh(&dc->lock); cookie = dma_cookie_assign(tx); - dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u %p\n", + dev_vdbg(dmaengine_chan_dev(tx->chan), "tx_submit: queued %u %p\n", desc->txd.cookie, desc); list_add_tail(&desc->desc_node, &dc->queue); @@ -717,11 +712,11 @@ txx9dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t xfer_count; size_t offset; - dev_vdbg(chan2dev(chan), "prep_dma_memcpy d%#llx s%#llx l%#zx f%#lx\n", + dev_vdbg(dmaengine_chan_dev(chan), "prep_dma_memcpy d%#llx s%#llx l%#zx f%#lx\n", (u64)dest, (u64)src, len, flags); if (unlikely(!len)) { - dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); + dev_dbg(dmaengine_chan_dev(chan), "prep_dma_memcpy: length is zero!\n"); return NULL; } @@ -812,7 +807,7 @@ txx9dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int i; struct scatterlist *sg; - dev_vdbg(chan2dev(chan), "prep_dma_slave\n"); + dev_vdbg(dmaengine_chan_dev(chan), "prep_dma_slave\n"); BUG_ON(!ds || !ds->reg_width); if (ds->tx_reg) @@ -900,7 +895,7 @@ static int txx9dmac_terminate_all(struct dma_chan *chan) struct txx9dmac_desc *desc, *_desc; LIST_HEAD(list); - dev_vdbg(chan2dev(chan), "terminate_all\n"); + dev_vdbg(dmaengine_chan_dev(chan), "terminate_all\n"); spin_lock_bh(&dc->lock); txx9dmac_reset_chan(dc); @@ -988,11 +983,11 @@ static int txx9dmac_alloc_chan_resources(struct dma_chan *chan) struct txx9dmac_desc *desc; int i; - dev_vdbg(chan2dev(chan), "alloc_chan_resources\n"); + dev_vdbg(dmaengine_chan_dev(chan), "alloc_chan_resources\n"); /* ASSERT: channel is idle */ if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) { - dev_dbg(chan2dev(chan), "DMA channel not idle?\n"); + dev_dbg(dmaengine_chan_dev(chan), "DMA channel not idle?\n"); return -EIO; } @@ -1022,7 +1017,7 @@ static int txx9dmac_alloc_chan_resources(struct dma_chan *chan) desc = txx9dmac_desc_alloc(dc, GFP_KERNEL); if (!desc) { - dev_info(chan2dev(chan), + dev_info(dmaengine_chan_dev(chan), "only allocated %d descriptors\n", i); spin_lock_bh(&dc->lock); break; @@ -1034,7 +1029,7 @@ static int txx9dmac_alloc_chan_resources(struct dma_chan *chan) } spin_unlock_bh(&dc->lock); - dev_dbg(chan2dev(chan), + dev_dbg(dmaengine_chan_dev(chan), "alloc_chan_resources allocated %d descriptors\n", i); return i; @@ -1047,7 +1042,7 @@ static void txx9dmac_free_chan_resources(struct dma_chan *chan) struct txx9dmac_desc *desc, *_desc; LIST_HEAD(list); - dev_dbg(chan2dev(chan), "free_chan_resources (descs allocated=%u)\n", + dev_dbg(dmaengine_chan_dev(chan), "free_chan_resources (descs allocated=%u)\n", dc->descs_allocated); /* ASSERT: channel is idle */ @@ -1061,13 +1056,13 @@ static void txx9dmac_free_chan_resources(struct dma_chan *chan) spin_unlock_bh(&dc->lock); list_for_each_entry_safe(desc, _desc, &list, desc_node) { - dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc); + dev_vdbg(dmaengine_chan_dev(chan), " freeing descriptor %p\n", desc); dma_unmap_single(chan2parent(chan), desc->txd.phys, ddev->descsize, DMA_TO_DEVICE); kfree(desc); } - dev_vdbg(chan2dev(chan), "free_chan_resources done\n"); + dev_vdbg(dmaengine_chan_dev(chan), "free_chan_resources done\n"); } /*----------------------------------------------------------------------*/ @@ -1167,26 +1162,20 @@ static void txx9dmac_chan_remove(struct platform_device *pdev) static int __init txx9dmac_probe(struct platform_device *pdev) { struct txx9dmac_platform_data *pdata = dev_get_platdata(&pdev->dev); - struct resource *io; struct txx9dmac_dev *ddev; + void __iomem *regs; u32 mcr; int err; - io = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!io) - return -EINVAL; + regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(regs)) + return PTR_ERR(regs); ddev = devm_kzalloc(&pdev->dev, sizeof(*ddev), GFP_KERNEL); if (!ddev) return -ENOMEM; - if (!devm_request_mem_region(&pdev->dev, io->start, resource_size(io), - dev_name(&pdev->dev))) - return -EBUSY; - - ddev->regs = devm_ioremap(&pdev->dev, io->start, resource_size(io)); - if (!ddev->regs) - return -ENOMEM; + ddev->regs = regs; ddev->have_64bit_regs = pdata->have_64bit_regs; if (__is_dmac64(ddev)) ddev->descsize = sizeof(struct txx9dmac_hwdesc); diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h index 59d9eabc8b67..abf2ae737c30 100644 --- a/drivers/dma/virt-dma.h +++ b/drivers/dma/virt-dma.h @@ -41,6 +41,11 @@ static inline struct virt_dma_chan *to_virt_chan(struct dma_chan *chan) return container_of(chan, struct virt_dma_chan, chan); } +static inline struct device *vchan_chan_dev(struct virt_dma_chan *vc) +{ + return dmaengine_chan_dev(&vc->chan); +} + void vchan_dma_desc_free_list(struct virt_dma_chan *vc, struct list_head *head); void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev); struct virt_dma_desc *vchan_find_desc(struct virt_dma_chan *, dma_cookie_t); @@ -209,6 +214,17 @@ static inline void vchan_free_chan_resources(struct virt_dma_chan *vc) vchan_dma_desc_free_list(vc, &head); } +/** + * vchan_chan_name - Return vchan DMA channel device name + * @vc: virtual channel + * + * Return: The name of the DMA channel device + */ +static inline const char *vchan_chan_name(struct virt_dma_chan *vc) +{ + return dma_chan_name(&vc->chan); +} + /** * vchan_synchronize() - synchronize callback execution to the current context * @vc: virtual channel to synchronize diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c index fa1173e49900..8d1c58ca0d48 100644 --- a/drivers/dma/xgene-dma.c +++ b/drivers/dma/xgene-dma.c @@ -1800,8 +1800,8 @@ static void xgene_dma_remove(struct platform_device *pdev) #ifdef CONFIG_ACPI static const struct acpi_device_id xgene_dma_acpi_match_ptr[] = { - {"APMC0D43", 0}, - {}, + { .id = "APMC0D43" }, + { } }; MODULE_DEVICE_TABLE(acpi, xgene_dma_acpi_match_ptr); #endif diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index cffe7c6fa640..0336941d4095 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -223,6 +223,7 @@ #define XILINX_MCDMA_IRQ_ERR_MASK BIT(7) #define XILINX_MCDMA_BD_EOP BIT(30) #define XILINX_MCDMA_BD_SOP BIT(31) +#define XILINX_MCDMA_BD_HW_SIZE 64 /** * struct xilinx_vdma_desc_hw - Hardware Descriptor @@ -277,8 +278,10 @@ struct xilinx_axidma_desc_hw { * @buf_addr_msb: MSB of Buffer address @0x0C * @rsvd: Reserved field @0x10 * @control: Control Information field @0x14 - * @status: Status field @0x18 - * @sideband_status: Status of sideband signals @0x1C + * @mm2s_ctrl_sideband: Sideband control info for mm2s @0x18 + * @s2mm_status: Status field for s2mm @0x18 + * @mm2s_status: Status field for mm2s @0x1C + * @s2mm_sideband_status: Sideband status for s2mm @0x1C * @app: APP Fields @0x20 - 0x30 */ struct xilinx_aximcdma_desc_hw { @@ -288,10 +291,17 @@ struct xilinx_aximcdma_desc_hw { u32 buf_addr_msb; u32 rsvd; u32 control; - u32 status; - u32 sideband_status; + union { + u32 mm2s_ctrl_sideband; + u32 s2mm_status; + }; + union { + u32 mm2s_status; + u32 s2mm_sideband_status; + }; u32 app[XILINX_DMA_NUM_APP_WORDS]; } __aligned(64); +static_assert(sizeof(struct xilinx_aximcdma_desc_hw) == XILINX_MCDMA_BD_HW_SIZE); /** * struct xilinx_cdma_desc_hw - Hardware Descriptor @@ -641,17 +651,51 @@ static inline void xilinx_aximcdma_buf(struct xilinx_dma_chan *chan, * @tx: async transaction descriptor * @payload_len: metadata payload length * @max_len: metadata max length - * Return: The app field pointer. + * + * The metadata lives in the SOP descriptor for TX and the EOF descriptor for RX. + * Field order depends on dmatype and direction: + * + * AXI DMA: [0..] app + * AXI MCDMA (TX): [0] ctrl_sideband, [1] status, [2..] app + * AXI MCDMA (RX): [0] status, [1] sideband, [2..] app + * + * Return: Pointer to the first metadata word. */ static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx, size_t *payload_len, size_t *max_len) { struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx); + struct xilinx_dma_chan *chan = to_xilinx_chan(tx->chan); + + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { + struct xilinx_aximcdma_tx_segment *seg; + + if (chan->direction == DMA_DEV_TO_MEM) { + seg = list_last_entry(&desc->segments, + struct xilinx_aximcdma_tx_segment, node); + *max_len = *payload_len = sizeof(seg->hw.s2mm_status) + + sizeof(seg->hw.s2mm_sideband_status) + + sizeof(seg->hw.app); + return &seg->hw.s2mm_status; + } + seg = list_first_entry(&desc->segments, + struct xilinx_aximcdma_tx_segment, node); + *max_len = *payload_len = sizeof(seg->hw.mm2s_ctrl_sideband) + + sizeof(seg->hw.mm2s_status) + + sizeof(seg->hw.app); + return &seg->hw.mm2s_ctrl_sideband; + } + struct xilinx_axidma_tx_segment *seg; - *max_len = *payload_len = sizeof(u32) * XILINX_DMA_NUM_APP_WORDS; - seg = list_first_entry(&desc->segments, - struct xilinx_axidma_tx_segment, node); + if (chan->direction == DMA_DEV_TO_MEM) + seg = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + else + seg = list_first_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + + *max_len = *payload_len = sizeof(seg->hw.app); return seg->hw.app; } @@ -1025,9 +1069,11 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, struct xilinx_aximcdma_tx_segment, node); aximcdma_hw = &aximcdma_seg->hw; - residue += - (aximcdma_hw->control & chan->xdev->max_buffer_len) - - (aximcdma_hw->status & chan->xdev->max_buffer_len); + residue += aximcdma_hw->control & chan->xdev->max_buffer_len; + if (chan->direction == DMA_DEV_TO_MEM) + residue -= aximcdma_hw->s2mm_status & chan->xdev->max_buffer_len; + else + residue -= aximcdma_hw->mm2s_status & chan->xdev->max_buffer_len; } } @@ -1782,6 +1828,17 @@ static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan) struct xilinx_axidma_tx_segment, node); if (!(seg->hw.status & XILINX_DMA_BD_COMP_MASK) && chan->has_sg) break; + } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { + struct xilinx_aximcdma_tx_segment *seg; + u32 status; + + seg = list_last_entry(&desc->segments, + struct xilinx_aximcdma_tx_segment, + node); + status = (chan->direction == DMA_DEV_TO_MEM) ? + seg->hw.s2mm_status : seg->hw.mm2s_status; + if (!(status & XILINX_DMA_BD_COMP_MASK)) + break; } if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA) @@ -2626,6 +2683,9 @@ xilinx_mcdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, segment->hw.control |= XILINX_MCDMA_BD_EOP; } + if (chan->xdev->has_axistream_connected) + desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; + return &desc->async_tx; error: @@ -3274,7 +3334,8 @@ static int xilinx_dma_probe(struct platform_device *pdev) dma_set_max_seg_size(xdev->dev, xdev->max_buffer_len); - if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA || + xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { xdev->has_axistream_connected = of_property_read_bool(node, "xlnx,axistream-connected"); } diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index 2b03584102ac..3b7dfa53f972 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -112,8 +113,7 @@ #define ZYNQMP_DMA_INT_DONE (ZYNQMP_DMA_DONE | ZYNQMP_DMA_DST_DSCR_DONE) #define ZYNQMP_DMA_INT_EN_DEFAULT_MASK (ZYNQMP_DMA_INT_DONE | \ ZYNQMP_DMA_INT_ERR | \ - ZYNQMP_DMA_INT_OVRFL | \ - ZYNQMP_DMA_DST_DSCR_DONE) + ZYNQMP_DMA_INT_OVRFL) /* Max number of descriptors per channel */ #define ZYNQMP_DMA_NUM_DESCS 32 @@ -128,8 +128,6 @@ /* Reset values for data attributes */ #define ZYNQMP_DMA_AXCACHE_VAL 0xF -#define ZYNQMP_DMA_SRC_ISSUE_RST_VAL 0x1F - #define ZYNQMP_DMA_IDS_DEFAULT_MASK 0xFFF /* Bus width in bits */ @@ -205,10 +203,10 @@ struct zynqmp_dma_desc_sw { * @desc_pool_p: Physical allocated descriptor base * @desc_free_cnt: Descriptor available count * @dev: The dma device - * @irq: Channel IRQ + * @irq: Linux IRQ number, or -1 when not registered * @is_dmacoherent: Tells whether dma operations are coherent or not * @tasklet: Cleanup work after irq - * @idle : Channel status; + * @idle: Channel status * @desc_size: Size of the low level descriptor * @err: Channel has errors * @bus_width: Bus width @@ -435,7 +433,7 @@ zynqmp_dma_get_descriptor(struct zynqmp_dma_chan *chan) } /** - * zynqmp_dma_free_descriptor - Issue pending transactions + * zynqmp_dma_free_descriptor - Return a descriptor to the free pool * @chan: ZynqMP DMA channel pointer * @sdesc: Transaction descriptor pointer */ @@ -483,8 +481,10 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan) return ret; chan->sw_desc_pool = kzalloc_objs(*desc, ZYNQMP_DMA_NUM_DESCS); - if (!chan->sw_desc_pool) - return -ENOMEM; + if (!chan->sw_desc_pool) { + ret = -ENOMEM; + goto err_pm; + } chan->idle = true; chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS; @@ -502,8 +502,10 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan) (2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS), &chan->desc_pool_p, GFP_KERNEL); - if (!chan->desc_pool_v) - return -ENOMEM; + if (!chan->desc_pool_v) { + ret = -ENOMEM; + goto err_free_sw_desc_pool; + } for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) { desc = chan->sw_desc_pool + i; @@ -516,6 +518,13 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan) } return ZYNQMP_DMA_NUM_DESCS; + +err_free_sw_desc_pool: + kfree(chan->sw_desc_pool); + chan->sw_desc_pool = NULL; +err_pm: + pm_runtime_put_autosuspend(chan->dev); + return ret; } /** @@ -606,7 +615,6 @@ static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan) zynqmp_dma_start(chan); } - /** * zynqmp_dma_chan_desc_cleanup - Cleanup the completed descriptors * @chan: ZynqMP DMA channel @@ -826,7 +834,8 @@ static void zynqmp_dma_synchronize(struct dma_chan *dchan) * @len: Transfer length * @flags: transfer ack flags * - * Return: Async transaction descriptor on success and NULL on failure + * Return: Async transaction descriptor on success and NULL on failure or + * zero length transfer */ static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy( struct dma_chan *dchan, dma_addr_t dma_dst, @@ -841,6 +850,9 @@ static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy( chan = to_chan(dchan); + if (!len) + return NULL; + desc_cnt = DIV_ROUND_UP(len, ZYNQMP_DMA_MAX_TRANS_LEN); spin_lock_irqsave(&chan->lock, irqflags); @@ -885,10 +897,11 @@ static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan) if (!chan) return; - if (chan->irq) + if (chan->irq >= 0) devm_free_irq(chan->zdev->dev, chan->irq, chan); tasklet_kill(&chan->tasklet); - list_del(&chan->common.device_node); + if (!list_empty(&chan->common.device_node)) + list_del(&chan->common.device_node); } /** @@ -896,7 +909,7 @@ static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan) * @zdev: Driver specific device structure * @pdev: Pointer to the platform_device structure * - * Return: '0' on success and failure value on error + * Return: 0 on success and negative error code on failure */ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, struct platform_device *pdev) @@ -904,13 +917,14 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, struct zynqmp_dma_chan *chan; struct device_node *node = pdev->dev.of_node; const struct zynqmp_dma_config *match_data; - int err; + int err, ret; chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL); if (!chan) return -ENOMEM; chan->dev = zdev->dev; chan->zdev = zdev; + chan->irq = -1; chan->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(chan->regs)) @@ -935,7 +949,7 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, if (match_data) chan->irq_offset = match_data->offset; - chan->is_dmacoherent = of_property_read_bool(node, "dma-coherent"); + chan->is_dmacoherent = of_dma_is_coherent(node); zdev->chan = chan; tasklet_setup(&chan->tasklet, zynqmp_dma_do_tasklet); spin_lock_init(&chan->lock); @@ -943,22 +957,27 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, INIT_LIST_HEAD(&chan->pending_list); INIT_LIST_HEAD(&chan->done_list); INIT_LIST_HEAD(&chan->free_list); + INIT_LIST_HEAD(&chan->common.device_node); dma_cookie_init(&chan->common); chan->common.device = &zdev->common; - list_add_tail(&chan->common.device_node, &zdev->common.channels); zynqmp_dma_init(chan); - chan->irq = platform_get_irq(pdev, 0); - if (chan->irq < 0) - return -ENXIO; - err = devm_request_irq(&pdev->dev, chan->irq, zynqmp_dma_irq_handler, 0, + ret = platform_get_irq(pdev, 0); + if (ret < 0) + return ret; + + err = devm_request_irq(&pdev->dev, ret, zynqmp_dma_irq_handler, 0, "zynqmp-dma", chan); if (err) return err; + chan->irq = ret; + chan->desc_size = sizeof(struct zynqmp_dma_desc_ll); chan->idle = true; + list_add_tail(&chan->common.device_node, &zdev->common.channels); + return 0; } @@ -1025,11 +1044,11 @@ static int __maybe_unused zynqmp_dma_runtime_suspend(struct device *dev) } /** - * zynqmp_dma_runtime_resume - Runtime suspend method for the driver + * zynqmp_dma_runtime_resume - Runtime resume method for the driver * @dev: Address of the device structure * - * Put the driver into low power mode. - * Return: 0 always + * Enable device clocks. + * Return: 0 on success and failure value on error */ static int __maybe_unused zynqmp_dma_runtime_resume(struct device *dev) { @@ -1123,7 +1142,7 @@ static int zynqmp_dma_probe(struct platform_device *pdev) ret = zynqmp_dma_chan_probe(zdev, pdev); if (ret) { dev_err_probe(&pdev->dev, ret, "Probing channel failed\n"); - goto err_disable_pm; + goto free_chan_resources; } p->dst_addr_widths = BIT(zdev->chan->bus_width / 8); @@ -1149,7 +1168,6 @@ static int zynqmp_dma_probe(struct platform_device *pdev) free_chan_resources: zynqmp_dma_chan_remove(zdev->chan); -err_disable_pm: if (!pm_runtime_enabled(zdev->dev)) zynqmp_dma_runtime_suspend(zdev->dev); pm_runtime_disable(zdev->dev); diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 074b007dc296..12f102f48d5d 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -1041,6 +1041,7 @@ static void dw_pcie_edma_init_data(struct dw_pcie *pci) pci->edma.ops = &dw_pcie_edma_ops; pci->edma.flags |= DW_EDMA_CHIP_LOCAL; + pci->edma.ch_space_sz = 256; } static int dw_pcie_edma_find_mf(struct dw_pcie *pci) diff --git a/include/dt-bindings/dma/amlogic,a9-dma.h b/include/dt-bindings/dma/amlogic,a9-dma.h new file mode 100644 index 000000000000..ea7c90063572 --- /dev/null +++ b/include/dt-bindings/dma/amlogic,a9-dma.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ + +#ifndef __DT_BINDINGS_DMA_AMLOGIC_DMA_H__ +#define __DT_BINDINGS_DMA_AMLOGIC_DMA_H__ + +#define AMLOGIC_A9_DMA_TX 0 +#define AMLOGIC_A9_DMA_RX 1 +#endif /* __DT_BINDINGS_DMA_AMLOGIC_DMA_H__ */ diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index 3c8e2ef9dee0..eead04b805e7 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -101,6 +101,27 @@ enum dw_edma_ch_irq_mode { DW_EDMA_CH_IRQ_REMOTE, }; +#define DW_EDMA_CH_CONFIG_NON_LL BIT(0) +#define DW_EDMA_CH_CONFIG_IRQ_MODE BIT(1) + +/** + * struct dw_edma_chan_config - dw-edma channel configuration + * @flags: fields selected by DW_EDMA_CH_CONFIG_* + * @non_ll: use HDMA non-linked-list mode + * @irq_mode: interrupt routing mode + * + * Pass this structure through dma_slave_config.peripheral_config. Before + * synchronizing a remote-routed channel, the client must stop remote + * programming and own every channel affected by the hardware quiesce: the + * entire direction for eDMA-compatible layouts, or the individual channel for + * native HDMA. + */ +struct dw_edma_chan_config { + u32 flags; + bool non_ll; + enum dw_edma_ch_irq_mode irq_mode; +}; + /** * struct dw_edma_chip - representation of DesignWare eDMA controller hardware * @dev: struct device of the eDMA controller @@ -130,7 +151,7 @@ struct dw_edma_chip { u16 ll_wr_cnt; u16 ll_rd_cnt; - /* link list address */ + /* Linked-list regions must not cross a 4 GiB address boundary. */ struct dw_edma_region ll_region_wr[HDMA_MAX_WR_CH]; struct dw_edma_region ll_region_rd[HDMA_MAX_RD_CH]; @@ -147,6 +168,7 @@ struct dw_edma_chip { struct dw_edma *dw; bool cfg_non_ll; + u32 ch_space_sz; }; /* Export to the platform drivers */ diff --git a/include/linux/dma/imx-dma.h b/include/linux/dma/imx-dma.h index 76a8de9ae151..7c678316b757 100644 --- a/include/linux/dma/imx-dma.h +++ b/include/linux/dma/imx-dma.h @@ -57,15 +57,12 @@ struct imx_dma_data { int priority; }; -static inline int imx_dma_is_ipu(struct dma_chan *chan) -{ - return !strcmp(dev_name(chan->device->dev), "ipu-core"); -} - static inline int imx_dma_is_general_purpose(struct dma_chan *chan) { - return !strcmp(chan->device->dev->driver->name, "imx-sdma") || - !strcmp(chan->device->dev->driver->name, "imx-dma"); + struct device *dev = dmaengine_get_provider_device(chan); + + return !strcmp(dev->driver->name, "imx-sdma") || + !strcmp(dev->driver->name, "imx-dma"); } /** diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index f322e501ff62..805668fbe437 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -325,7 +325,8 @@ struct dma_router { * @lock: protect between config and prepare transfer when driver have not * implemented callback device_prep_config_sg(). * @chan_id: channel ID for sysfs - * @dev: class device for sysfs + * @chan_dev: class channel device for sysfs, some device use it for per-channel + * IOMMU mapping. * @name: backlink name for sysfs * @dbg_client_name: slave name for debugfs in format: * dev_name(requester's dev):channel name, for example: "2b00000.mcasp:tx" @@ -351,7 +352,14 @@ struct dma_chan { /* sysfs */ int chan_id; - struct dma_chan_dev *dev; + union { + struct dma_chan_dev *chan_dev; + /* + * Use chan_dev; dev will be removed once all users are + * converted. + */ + struct dma_chan_dev *dev; + }; const char *name; #ifdef CONFIG_DEBUG_FS char *dbg_client_name; @@ -369,6 +377,26 @@ struct dma_chan { void *private; }; +#define DMA_CHAN_ID_STATIC BIT(30) + +/** + * dmaengine_set_static_chan_id - request an exact DMA engine channel ID + * @chan: DMA channel + * @id: channel ID, unique within the DMA device + * + * Drivers may call this after initializing @chan and before registering its + * DMA device. The dmaengine core reserves @id from the device IDA instead of + * assigning the next available ID. + */ +static inline void dmaengine_set_static_chan_id(struct dma_chan *chan, + unsigned int id) +{ + if (WARN_ON_ONCE(id >= DMA_CHAN_ID_STATIC)) + return; + + chan->chan_id = DMA_CHAN_ID_STATIC | id; +} + /** * struct dma_chan_dev - relate sysfs device node to backing channel device * @chan: driver channel device @@ -532,7 +560,7 @@ struct dma_slave_caps { static inline const char *dma_chan_name(struct dma_chan *chan) { - return dev_name(&chan->dev->device); + return dev_name(&chan->chan_dev->device); } /** @@ -981,12 +1009,6 @@ static inline int dmaengine_slave_config(struct dma_chan *chan, return -ENOSYS; } -static inline bool is_slave_direction(enum dma_transfer_direction direction) -{ - return (direction == DMA_MEM_TO_DEV) || (direction == DMA_DEV_TO_MEM) || - (direction == DMA_DEV_TO_DEV); -} - static inline struct dma_async_tx_descriptor * dmaengine_prep_config_single(struct dma_chan *chan, dma_addr_t buf, size_t len, enum dma_transfer_direction dir, @@ -1224,6 +1246,137 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memcpy( len, flags); } +/** + * dmaengine_get_max_xor - get the maximum number of XOR sources for a channel + * @chan: DMA channel + * + * Returns the maximum number of source buffers supported by the XOR engine + * behind @chan, or 0 if the channel does not support XOR operations. + */ +static inline unsigned int dmaengine_get_max_xor(struct dma_chan *chan) +{ + if (!chan || !chan->device) + return 0; + + return chan->device->max_xor; +} + +/** + * dmaengine_prep_dma_xor - prepare a DMA XOR operation + * @chan: the channel to use for this operation + * @dst: destination buffer address + * @src: array of source buffer addresses + * @src_cnt: number of source buffers + * @len: length in bytes of each source and destination buffer + * @flags: DMA engine flags (e.g. DMA_PREP_INTERRUPT) + * + * Prepare an XOR parity generation transaction. The engine computes: + * dst = src[0] XOR src[1] XOR ... XOR src[src_cnt - 1] + * + * Returns a descriptor on success, or NULL if the channel does not support + * this operation or the request could not be queued. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_xor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, + unsigned int src_cnt, size_t len, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_xor) + return NULL; + + return chan->device->device_prep_dma_xor(chan, dst, src, + src_cnt, len, flags); +} + +/** + * dmaengine_prep_dma_xor_val - prepare a DMA XOR zero-sum validation operation + * @chan: the channel to use for this operation + * @src: array of source buffer addresses + * @src_cnt: number of source buffers + * @len: length in bytes of each source buffer + * @result: output flag set to SUM_CHECK_P_RESULT if the XOR of all sources + * is non-zero (i.e. parity error), cleared otherwise + * @flags: DMA engine flags (e.g. DMA_PREP_INTERRUPT) + * + * Prepare an XOR zero-sum validation transaction. The engine XORs all + * source buffers and checks whether the result is zero. The outcome is + * written to @result on completion. + * + * Returns a descriptor on success, or NULL if the channel does not support + * this operation or the request could not be queued. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *src, + unsigned int src_cnt, size_t len, + enum sum_check_flags *result, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_xor_val) + return NULL; + + return chan->device->device_prep_dma_xor_val(chan, src, src_cnt, + len, result, flags); +} + +/** + * dmaengine_prep_dma_pq - prepare a DMA PQ (RAID-6 P+Q) operation + * @chan: the channel to use for this operation + * @dst: array of two destination addresses: dst[0] for P, dst[1] for Q + * @src: array of source buffer addresses + * @src_cnt: number of source buffers + * @scf: array of scaling coefficients, one per source buffer + * @len: length in bytes of each source and destination buffer + * @flags: DMA engine flags (e.g. DMA_PREP_INTERRUPT) + * + * Prepare a P+Q parity generation transaction. The engine computes: + * P = XOR of all source buffers + * Q = Galois field sum of (scf[i] * src[i]) over all sources + * + * Returns a descriptor on success, or NULL if the channel does not support + * this operation or the request could not be queued. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, + size_t len, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_pq) + return NULL; + + return chan->device->device_prep_dma_pq(chan, dst, src, + src_cnt, scf, len, flags); +} + +/** + * dmaengine_prep_dma_pq_val - prepare a DMA PQ validation operation + * @chan: the channel to use for this operation + * @pq: array of two addresses holding existing P and Q parity buffers + * @src: array of source buffer addresses + * @src_cnt: number of source buffers + * @scf: array of scaling coefficients, one per source buffer + * @len: length in bytes of each buffer + * @pqres: output flags indicating P and/or Q check results (SUM_CHECK_P_VALID, + * SUM_CHECK_Q_VALID) + * @flags: DMA engine flags (e.g. DMA_PREP_INTERRUPT) + * + * Prepare a PQ validation transaction. The engine recomputes P and Q from the + * source buffers and compares them against the existing parity stored at @pq. + * The result of each comparison is reported through @pqres. + * + * Returns a descriptor on success, or NULL if the channel does not support + * this operation or the request could not be queued. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, + size_t len, enum sum_check_flags *pqres, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_pq_val) + return NULL; + + return chan->device->device_prep_dma_pq_val(chan, pq, src, + src_cnt, scf, len, + pqres, flags); +} + static inline bool dmaengine_is_metadata_mode_supported(struct dma_chan *chan, enum dma_desc_metadata_mode mode) { @@ -1413,6 +1566,130 @@ static inline bool is_dma_fill_aligned(struct dma_device *dev, size_t off1, return dmaengine_check_align(dev->fill_align, off1, off2, len); } +/** + * dmaengine_get_copy_align - get copy alignment requirement of a DMA channel + * @chan: DMA channel + * + * Return the copy alignment requirement of @chan as a power-of-2 exponent. + */ +static inline enum dmaengine_alignment +dmaengine_get_copy_align(struct dma_chan *chan) +{ + return chan->device->copy_align; +} + +/** + * dmaengine_get_xor_align - get xor alignment requirement of a DMA channel + * @chan: DMA channel + * + * Return the xor alignment requirement of @chan as a power-of-2 exponent. + */ +static inline enum dmaengine_alignment +dmaengine_get_xor_align(struct dma_chan *chan) +{ + return chan->device->xor_align; +} + +/** + * dmaengine_get_pq_align - get pq alignment requirement of a DMA channel + * @chan: DMA channel + * + * Return the pq alignment requirement of @chan as a power-of-2 exponent. + */ +static inline enum dmaengine_alignment +dmaengine_get_pq_align(struct dma_chan *chan) +{ + return chan->device->pq_align; +} + +/** + * dmaengine_get_fill_align - get fill alignment requirement of a DMA channel + * @chan: DMA channel + * + * Return the fill alignment requirement of @chan as a power-of-2 exponent. + */ +static inline enum dmaengine_alignment +dmaengine_get_fill_align(struct dma_chan *chan) +{ + return chan->device->fill_align; +} + +/** + * dmaengine_get_cap_mask - get capability mask of a DMA channel + * @chan: DMA channel + * + * Return a pointer to the capability mask of the DMA device backing @chan. + */ +static inline const dma_cap_mask_t * +dmaengine_get_cap_mask(struct dma_chan *chan) +{ + return &chan->device->cap_mask; +} + +/** + * dmaengine_is_copy_aligned - test copy alignment + * @chan: DMA channel + * @off1: first buffer offset + * @off2: second buffer offset + * @len: transfer length + * + * Return true if off1, off2 and len satisfy the copy alignment requirement of + * the DMA channel, false otherwise. + */ +static inline bool dmaengine_is_copy_aligned(struct dma_chan *chan, size_t off1, + size_t off2, size_t len) +{ + return dmaengine_check_align(chan->device->copy_align, off1, off2, len); +} + +/** + * dmaengine_is_xor_aligned - test xor alignment + * @chan: DMA channel + * @off1: first buffer offset + * @off2: second buffer offset + * @len: transfer length + * + * Return true if off1, off2 and len satisfy the xor alignment requirement of + * the DMA channel, false otherwise. + */ +static inline bool dmaengine_is_xor_aligned(struct dma_chan *chan, size_t off1, + size_t off2, size_t len) +{ + return dmaengine_check_align(chan->device->xor_align, off1, off2, len); +} + +/** + * dmaengine_is_pq_aligned - test pq alignment + * @chan: DMA channel + * @off1: first buffer offset + * @off2: second buffer offset + * @len: transfer length + * + * Return true if off1, off2 and len satisfy the pq alignment requirement of + * the DMA channel, false otherwise. + */ +static inline bool dmaengine_is_pq_aligned(struct dma_chan *chan, size_t off1, + size_t off2, size_t len) +{ + return dmaengine_check_align(chan->device->pq_align, off1, off2, len); +} + +/** + * dmaengine_is_fill_aligned - test fill alignment + * @chan: DMA channel + * @off1: first buffer offset + * @off2: second buffer offset + * @len: transfer length + * + * Return true if off1, off2 and len satisfy the fill alignment requirement of + * the DMA channel, false otherwise. + */ +static inline bool dmaengine_is_fill_aligned(struct dma_chan *chan, size_t off1, + size_t off2, size_t len) +{ + return dmaengine_check_align(chan->device->fill_align, off1, off2, len); +} + static inline void dma_set_maxpq(struct dma_device *dma, int maxpq, int has_pq_continue) { @@ -1574,9 +1851,40 @@ __dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp) return test_bit(tx_type, srcp->bits); } +/** + * dmaengine_has_cap - test whether a DMA channel supports a transaction type + * @chan: DMA channel + * @tx_type: transaction type to test + * + * Return true if @chan supports @tx_type, false otherwise. + */ +static inline bool +dmaengine_has_cap(struct dma_chan *chan, enum dma_transaction_type tx_type) +{ + return dma_has_cap(tx_type, chan->device->cap_mask); +} + #define for_each_dma_cap_mask(cap, mask) \ for_each_set_bit(cap, mask.bits, DMA_TX_TYPE_END) +/** + * dmaengine_prep_dma_interrupt() - Prepare a DMA interrupt descriptor. + * @chan: The channel to be used for this descriptor + * @flags: DMA engine flags + * + * Returns a descriptor for an interrupt transaction, or NULL if the + * channel does not support DMA_INTERRUPT. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_interrupt || + !dma_has_cap(DMA_INTERRUPT, chan->device->cap_mask)) + return NULL; + + return chan->device->device_prep_dma_interrupt(chan, flags); +} + /** * dma_async_issue_pending - flush pending transactions to HW * @chan: target DMA channel @@ -1746,16 +2054,33 @@ static inline int dmaengine_desc_free(struct dma_async_tx_descriptor *desc) return desc->desc_free(desc); } +/** + * dmaengine_desc_set_callback - set the callback, callback_result and + * parameter on a descriptor + * @tx: the descriptor to set the callback on + * @cb: the plain completion callback, or NULL + * @cb_result: the result-bearing completion callback, or NULL + * @cb_param: the parameter to pass to the callback + * + * Sets the completion callbacks and their shared parameter on a DMA + * transaction descriptor. Use this helper instead of assigning the fields + * directly to ensure all three related fields are updated consistently. + * Only one of @cb and @cb_result should be non-NULL; if both are provided + * @cb_result takes precedence in drivers that support it. + */ +static inline void +dmaengine_desc_set_callback(struct dma_async_tx_descriptor *tx, + dma_async_tx_callback cb, + dma_async_tx_callback_result cb_result, + void *cb_param) +{ + tx->callback = cb; + tx->callback_result = cb_result; + tx->callback_param = cb_param; +} + /* --- DMA device --- */ -int dma_async_device_register(struct dma_device *device); -int dmaenginem_async_device_register(struct dma_device *device); -void dma_async_device_unregister(struct dma_device *device); -int dma_async_device_channel_register(struct dma_device *device, - struct dma_chan *chan, - const char *name); -void dma_async_device_channel_unregister(struct dma_device *device, - struct dma_chan *chan); void dma_run_dependencies(struct dma_async_tx_descriptor *tx); #define dma_request_channel(mask, x, y) \ __dma_request_channel(&(mask), x, y, NULL) @@ -1777,29 +2102,36 @@ static inline struct dma_chan return dma_request_channel(mask, fn, fn_param); } -static inline char * -dmaengine_get_direction_text(enum dma_transfer_direction dir) +static inline struct device *dmaengine_chan_dev(struct dma_chan *chan) { - switch (dir) { - case DMA_DEV_TO_MEM: - return "DEV_TO_MEM"; - case DMA_MEM_TO_DEV: - return "MEM_TO_DEV"; - case DMA_MEM_TO_MEM: - return "MEM_TO_MEM"; - case DMA_DEV_TO_DEV: - return "DEV_TO_DEV"; - default: - return "invalid"; - } + return &chan->chan_dev->device; } -static inline struct device *dmaengine_get_dma_device(struct dma_chan *chan) +/* + * dmaengine_get_provider_device() - Get DMA engine provider device. Typical + * use is to find a DMA channel by filter. + * @chan: DMA channel + * + * Return: Pointer to the DMA engine provider device. + */ +static inline struct device *dmaengine_get_provider_device(struct dma_chan *chan) { - if (chan->dev->chan_dma_dev) - return &chan->dev->device; - return chan->device->dev; } +/* + * dmaengine_get_dma_device() - Get DMA device used for DMA mapping. Some + * DMA engines provide per-channel iommu mapping. + * @chan: DMA channel + * + * Return: Pointer to the device used for DMA mapping. + */ +static inline struct device *dmaengine_get_dma_device(struct dma_chan *chan) +{ + if (chan->chan_dev->chan_dma_dev) + return dmaengine_chan_dev(chan); + + return dmaengine_get_provider_device(chan); +} + #endif /* DMAENGINE_H */ diff --git a/include/linux/fsl/bestcomm/gen_bd.h b/include/linux/fsl/bestcomm/gen_bd.h index aeb312a1cd00..7253d308d77a 100644 --- a/include/linux/fsl/bestcomm/gen_bd.h +++ b/include/linux/fsl/bestcomm/gen_bd.h @@ -16,10 +16,6 @@ struct bcom_gen_bd { }; -extern struct bcom_task * -bcom_gen_bd_rx_init(int queue_len, phys_addr_t fifo, - int initiator, int ipr, int maxbufsize); - extern int bcom_gen_bd_rx_reset(struct bcom_task *tsk); @@ -27,10 +23,6 @@ extern void bcom_gen_bd_rx_release(struct bcom_task *tsk); -extern struct bcom_task * -bcom_gen_bd_tx_init(int queue_len, phys_addr_t fifo, - int initiator, int ipr); - extern int bcom_gen_bd_tx_reset(struct bcom_task *tsk); diff --git a/include/trace/events/tegra_apb_dma.h b/include/trace/events/tegra_apb_dma.h index 6d9f5075baa3..c38117c127ef 100644 --- a/include/trace/events/tegra_apb_dma.h +++ b/include/trace/events/tegra_apb_dma.h @@ -11,7 +11,7 @@ TRACE_EVENT(tegra_dma_tx_status, TP_PROTO(struct dma_chan *dc, dma_cookie_t cookie, struct dma_tx_state *state), TP_ARGS(dc, cookie, state), TP_STRUCT__entry( - __string(chan, dev_name(&dc->dev->device)) + __string(chan, dma_chan_name(dc)) __field(dma_cookie_t, cookie) __field(__u32, residue) ), @@ -28,7 +28,7 @@ TRACE_EVENT(tegra_dma_complete_cb, TP_PROTO(struct dma_chan *dc, int count, void *ptr), TP_ARGS(dc, count, ptr), TP_STRUCT__entry( - __string(chan, dev_name(&dc->dev->device)) + __string(chan, dma_chan_name(dc)) __field(int, count) __field(void *, ptr) ), @@ -45,7 +45,7 @@ TRACE_EVENT(tegra_dma_isr, TP_PROTO(struct dma_chan *dc, int irq), TP_ARGS(dc, irq), TP_STRUCT__entry( - __string(chan, dev_name(&dc->dev->device)) + __string(chan, dma_chan_name(dc)) __field(int, irq) ), TP_fast_assign(