From 85fa189b2c829a005224cab3c2d88635dd36d777 Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Mon, 21 Feb 2022 18:47:21 +0100 Subject: [PATCH 01/14] media: imx: De-stage imx7-mipi-csis The imx7-mipi-csis driver is in a good state and can be destaged. Move the imx7-mipi-csis.c driver to the newly created drivers/media/platform/imx directory and plumb the related options in Kconfig and in Makefile. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Acked-by: Rui Miguel Silva Acked-by: Sakari Ailus Signed-off-by: Hans Verkuil --- MAINTAINERS | 2 +- drivers/media/platform/Kconfig | 1 + drivers/media/platform/Makefile | 1 + drivers/media/platform/imx/Kconfig | 24 +++++++++++++++++++ drivers/media/platform/imx/Makefile | 1 + .../platform}/imx/imx7-mipi-csis.c | 0 drivers/staging/media/imx/Makefile | 1 - 7 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 drivers/media/platform/imx/Kconfig create mode 100644 drivers/media/platform/imx/Makefile rename drivers/{staging/media => media/platform}/imx/imx7-mipi-csis.c (100%) diff --git a/MAINTAINERS b/MAINTAINERS index 83d27b57016f..5bdb8c881b0b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11891,8 +11891,8 @@ T: git git://linuxtv.org/media_tree.git F: Documentation/admin-guide/media/imx7.rst F: Documentation/devicetree/bindings/media/nxp,imx7-csi.yaml F: Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml +F: drivers/media/platform/imx/imx7-mipi-csis.c F: drivers/staging/media/imx/imx7-media-csi.c -F: drivers/staging/media/imx/imx7-mipi-csis.c MEDIA DRIVERS FOR HELENE M: Abylay Ospan diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig index 9fbdba0fd1e7..d9eeccffea69 100644 --- a/drivers/media/platform/Kconfig +++ b/drivers/media/platform/Kconfig @@ -171,6 +171,7 @@ source "drivers/media/platform/xilinx/Kconfig" source "drivers/media/platform/rcar-vin/Kconfig" source "drivers/media/platform/atmel/Kconfig" source "drivers/media/platform/sunxi/Kconfig" +source "drivers/media/platform/imx/Kconfig" config VIDEO_TI_CAL tristate "TI CAL (Camera Adaptation Layer) driver" diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile index 28eb4aadbf45..a9466c854610 100644 --- a/drivers/media/platform/Makefile +++ b/drivers/media/platform/Makefile @@ -20,6 +20,7 @@ obj-y += ti-vpe/ obj-$(CONFIG_VIDEO_MX2_EMMAPRP) += mx2_emmaprp.o obj-$(CONFIG_VIDEO_CODA) += coda/ +obj-$(CONFIG_VIDEO_IMX) += imx/ obj-$(CONFIG_VIDEO_IMX_PXP) += imx-pxp.o obj-$(CONFIG_VIDEO_IMX8_JPEG) += imx-jpeg/ diff --git a/drivers/media/platform/imx/Kconfig b/drivers/media/platform/imx/Kconfig new file mode 100644 index 000000000000..683863572c20 --- /dev/null +++ b/drivers/media/platform/imx/Kconfig @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0-only + +menuconfig VIDEO_IMX + bool "V4L2 capture drivers for NXP i.MX devices" + depends on ARCH_MXC || COMPILE_TEST + depends on VIDEO_DEV && VIDEO_V4L2 + help + Say yes here to enable support for capture drivers on i.MX SoCs. + Support for the single SoC features are selectable in the sub-menu + options. + +if VIDEO_IMX + +config VIDEO_IMX_MIPI_CSIS + tristate "MIPI CSI-2 CSIS receiver found on i.MX7 and i.MX8 models" + select MEDIA_CONTROLLER + select V4L2_FWNODE + select VIDEO_V4L2_SUBDEV_API + default n + help + Video4Linux2 sub-device driver for the MIPI CSI-2 CSIS receiver + v3.3/v3.6.3 found on some i.MX7 and i.MX8 SoCs. + +endif # VIDEO_IMX diff --git a/drivers/media/platform/imx/Makefile b/drivers/media/platform/imx/Makefile new file mode 100644 index 000000000000..ee272234c8d7 --- /dev/null +++ b/drivers/media/platform/imx/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_VIDEO_IMX_MIPI_CSIS) += imx7-mipi-csis.o diff --git a/drivers/staging/media/imx/imx7-mipi-csis.c b/drivers/media/platform/imx/imx7-mipi-csis.c similarity index 100% rename from drivers/staging/media/imx/imx7-mipi-csis.c rename to drivers/media/platform/imx/imx7-mipi-csis.c diff --git a/drivers/staging/media/imx/Makefile b/drivers/staging/media/imx/Makefile index 19c2fc54d424..d82be898145b 100644 --- a/drivers/staging/media/imx/Makefile +++ b/drivers/staging/media/imx/Makefile @@ -15,5 +15,4 @@ obj-$(CONFIG_VIDEO_IMX_CSI) += imx6-media-csi.o obj-$(CONFIG_VIDEO_IMX_CSI) += imx6-mipi-csi2.o obj-$(CONFIG_VIDEO_IMX7_CSI) += imx7-media-csi.o -obj-$(CONFIG_VIDEO_IMX7_CSI) += imx7-mipi-csis.o obj-$(CONFIG_VIDEO_IMX7_CSI) += imx8mq-mipi-csi2.o From 5b21b566b977155407bd0d0ea0d28018f33e6c12 Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Mon, 21 Feb 2022 18:47:22 +0100 Subject: [PATCH 02/14] media: imx: Rename imx7-mipi-csis.c to imx-mipi-csis.c Rename the imx7-mipi-csis.c driver to remove the reference to i.MX7. The driver is for an IP core found on i.MX7 and i.MX8 SoC, so do not specify a SoC version number in the driver name. Remove the references to the i.MX7 SoC in the driver symbols and expand the driver's header with more information about the IP core the driver controls. Also rename the associated bindings documentation. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Acked-by: Rui Miguel Silva Acked-by: Sakari Ailus Signed-off-by: Hans Verkuil --- Documentation/admin-guide/media/imx7.rst | 2 +- ...{nxp,imx7-mipi-csi2.yaml => nxp,imx-mipi-csi2.yaml} | 2 +- MAINTAINERS | 4 ++-- drivers/media/platform/imx/Makefile | 2 +- .../platform/imx/{imx7-mipi-csis.c => imx-mipi-csis.c} | 10 +++++++--- 5 files changed, 12 insertions(+), 8 deletions(-) rename Documentation/devicetree/bindings/media/{nxp,imx7-mipi-csi2.yaml => nxp,imx-mipi-csi2.yaml} (98%) rename drivers/media/platform/imx/{imx7-mipi-csis.c => imx-mipi-csis.c} (99%) diff --git a/Documentation/admin-guide/media/imx7.rst b/Documentation/admin-guide/media/imx7.rst index 4785ae8ac978..2fa27718f52a 100644 --- a/Documentation/admin-guide/media/imx7.rst +++ b/Documentation/admin-guide/media/imx7.rst @@ -33,7 +33,7 @@ reference manual [#f1]_. Entities -------- -imx7-mipi-csi2 +imx-mipi-csi2 -------------- This is the MIPI CSI-2 receiver entity. It has one sink pad to receive the pixel diff --git a/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml b/Documentation/devicetree/bindings/media/nxp,imx-mipi-csi2.yaml similarity index 98% rename from Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml rename to Documentation/devicetree/bindings/media/nxp,imx-mipi-csi2.yaml index e2e6e9aa0fe6..36b135bf9f2a 100644 --- a/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml +++ b/Documentation/devicetree/bindings/media/nxp,imx-mipi-csi2.yaml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/media/nxp,imx7-mipi-csi2.yaml# +$id: http://devicetree.org/schemas/media/nxp,imx-mipi-csi2.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# title: NXP i.MX7 and i.MX8 MIPI CSI-2 receiver diff --git a/MAINTAINERS b/MAINTAINERS index 5bdb8c881b0b..d919ea3ed250 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11889,9 +11889,9 @@ L: linux-media@vger.kernel.org S: Maintained T: git git://linuxtv.org/media_tree.git F: Documentation/admin-guide/media/imx7.rst +F: Documentation/devicetree/bindings/media/nxp,imx-mipi-csi2.yaml F: Documentation/devicetree/bindings/media/nxp,imx7-csi.yaml -F: Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml -F: drivers/media/platform/imx/imx7-mipi-csis.c +F: drivers/media/platform/imx/imx-mipi-csis.c F: drivers/staging/media/imx/imx7-media-csi.c MEDIA DRIVERS FOR HELENE diff --git a/drivers/media/platform/imx/Makefile b/drivers/media/platform/imx/Makefile index ee272234c8d7..f72bdbe8e6ef 100644 --- a/drivers/media/platform/imx/Makefile +++ b/drivers/media/platform/imx/Makefile @@ -1 +1 @@ -obj-$(CONFIG_VIDEO_IMX_MIPI_CSIS) += imx7-mipi-csis.o +obj-$(CONFIG_VIDEO_IMX_MIPI_CSIS) += imx-mipi-csis.o diff --git a/drivers/media/platform/imx/imx7-mipi-csis.c b/drivers/media/platform/imx/imx-mipi-csis.c similarity index 99% rename from drivers/media/platform/imx/imx7-mipi-csis.c rename to drivers/media/platform/imx/imx-mipi-csis.c index 388cfd012212..100722b61c69 100644 --- a/drivers/media/platform/imx/imx7-mipi-csis.c +++ b/drivers/media/platform/imx/imx-mipi-csis.c @@ -1,6 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Freescale i.MX7 SoC series MIPI-CSI V3.3 receiver driver + * Samsung CSIS MIPI CSI-2 receiver driver. + * + * The Samsung CSIS IP is a MIPI CSI-2 receiver found in various NXP i.MX7 and + * i.MX8 SoCs. The i.MX7 features version 3.3 of the IP, while i.MX8 features + * version 3.6.3. * * Copyright (C) 2019 Linaro Ltd * Copyright (C) 2015-2016 Freescale Semiconductor, Inc. All Rights Reserved. @@ -31,7 +35,7 @@ #include #include -#define CSIS_DRIVER_NAME "imx7-mipi-csis" +#define CSIS_DRIVER_NAME "imx-mipi-csis" #define CSIS_PAD_SINK 0 #define CSIS_PAD_SOURCE 1 @@ -1519,4 +1523,4 @@ module_platform_driver(mipi_csis_driver); MODULE_DESCRIPTION("i.MX7 & i.MX8 MIPI CSI-2 receiver driver"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:imx7-mipi-csi2"); +MODULE_ALIAS("platform:imx-mipi-csi2"); From 9babbbaaeb87dee9a9a10dd2923b173902acf7ea Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Mon, 21 Feb 2022 18:47:23 +0100 Subject: [PATCH 03/14] media: imx: imx7-media-csi: Use dual sampling for YUV 1X16 The CSI bridge should operate in dual components mode when it is connected to a pixel transmitter that transfers two components at a time in YUV 422 formats (16 bits, Y + U/V). Use the image format variants to determine if single or dual component mode should be used. Add a note to the TODO file to record that the list of supported formats should be restricted to the SoC model the CSI bridge is integrated on to avoid potential pipeline mis-configurations. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Acked-by: Rui Miguel Silva Acked-by: Sakari Ailus Signed-off-by: Hans Verkuil --- drivers/staging/media/imx/TODO | 25 +++++++++++++++++ drivers/staging/media/imx/imx7-media-csi.c | 32 +++++++++++++++++++--- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/drivers/staging/media/imx/TODO b/drivers/staging/media/imx/TODO index 06c94f20ecf8..5d3a337c8702 100644 --- a/drivers/staging/media/imx/TODO +++ b/drivers/staging/media/imx/TODO @@ -27,3 +27,28 @@ - i.MX7: all of the above, since it uses the imx media core - i.MX7: use Frame Interval Monitor + +- imx7-media-csi: Restrict the supported formats list to the SoC version. + + The imx7 CSI bridge can be configured to sample pixel components from the Rx + queue in single (8bpp) or double (16bpp) component modes. Image format + variants with different sample sizes (ie YUYV_2X8 vs YUYV_1X16) determine the + pixel components sampling size per each clock cycle and their packing mode + (see imx7_csi_configure() for details). + + As the imx7 CSI bridge can be interfaced with different IP blocks depending on + the SoC model it is integrated on, the Rx queue sampling size should match + the size of the samples transferred by the transmitting IP block. + + To avoid mis-configurations of the capture pipeline, the enumeration of the + supported formats should be restricted to match the pixel source transmitting + mode. + + Example: i.MX8MM SoC integrates the CSI bridge with the Samsung CSIS CSI-2 + receiver which operates in dual pixel sampling mode. The CSI bridge should + only expose the 1X16 formats variant which instructs it to operate in dual + pixel sampling mode. When the CSI bridge is instead integrated on an i.MX7, + which supports both serial and parallel input, it should expose both variants. + + This currently only applies to YUYV formats, but other formats might need + to be handled in the same way. diff --git a/drivers/staging/media/imx/imx7-media-csi.c b/drivers/staging/media/imx/imx7-media-csi.c index 32311fc0e2a4..8467a1491048 100644 --- a/drivers/staging/media/imx/imx7-media-csi.c +++ b/drivers/staging/media/imx/imx7-media-csi.c @@ -498,16 +498,40 @@ static void imx7_csi_configure(struct imx7_csi *csi) cr3 |= BIT_TWO_8BIT_SENSOR; cr18 |= BIT_MIPI_DATA_FORMAT_RAW14; break; + /* - * CSI-2 sources are supposed to use the 1X16 formats, but not - * all of them comply. Support both variants. + * The CSI bridge has a 16-bit input bus. Depending on the + * connected source, data may be transmitted with 8 or 10 bits + * per clock sample (in bits [9:2] or [9:0] respectively) or + * with 16 bits per clock sample (in bits [15:0]). The data is + * then packed into a 32-bit FIFO (as shown in figure 13-11 of + * the i.MX8MM reference manual rev. 3). + * + * The data packing in a 32-bit FIFO input word is controlled by + * the CR3 TWO_8BIT_SENSOR field (also known as SENSOR_16BITS in + * the i.MX8MM reference manual). When set to 0, data packing + * groups four 8-bit input samples (bits [9:2]). When set to 1, + * data packing groups two 16-bit input samples (bits [15:0]). + * + * The register field CR18 MIPI_DOUBLE_CMPNT also needs to be + * configured according to the input format for YUV 4:2:2 data. + * The field controls the gasket between the CSI-2 receiver and + * the CSI bridge. On i.MX7 and i.MX8MM, the field must be set + * to 1 when the CSIS outputs 16-bit samples. On i.MX8MQ, the + * gasket ignores the MIPI_DOUBLE_CMPNT bit and YUV 4:2:2 always + * uses 16-bit samples. Setting MIPI_DOUBLE_CMPNT in that case + * has no effect, but doesn't cause any issue. */ case MEDIA_BUS_FMT_UYVY8_2X8: - case MEDIA_BUS_FMT_UYVY8_1X16: case MEDIA_BUS_FMT_YUYV8_2X8: - case MEDIA_BUS_FMT_YUYV8_1X16: cr18 |= BIT_MIPI_DATA_FORMAT_YUV422_8B; break; + case MEDIA_BUS_FMT_UYVY8_1X16: + case MEDIA_BUS_FMT_YUYV8_1X16: + cr3 |= BIT_TWO_8BIT_SENSOR; + cr18 |= BIT_MIPI_DATA_FORMAT_YUV422_8B | + BIT_MIPI_DOUBLE_CMPNT; + break; } } From 8635b5bc2f6a27da8efca4c4a7bae717398092ed Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Mon, 21 Feb 2022 18:47:24 +0100 Subject: [PATCH 04/14] media: imx: imx-mipi-csis: Set PIXEL_MODE for YUV422 Bits 13 and 12 of the ISP_CONFIGn register configure the PIXEL_MODE which specifies the sampling size, in pixel component units, on the CSI-2 output data interface when data are transferred to memory. The register description in the chip manual specifies that DUAL mode should be used for YUV422 data. Signed-off-by: Jacopo Mondi Signed-off-by: Xavier Roumegue Reviewed-by: Laurent Pinchart Acked-by: Rui Miguel Silva Acked-by: Sakari Ailus Signed-off-by: Hans Verkuil --- drivers/media/platform/imx/imx-mipi-csis.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/imx/imx-mipi-csis.c b/drivers/media/platform/imx/imx-mipi-csis.c index 100722b61c69..5873ff74f737 100644 --- a/drivers/media/platform/imx/imx-mipi-csis.c +++ b/drivers/media/platform/imx/imx-mipi-csis.c @@ -173,6 +173,7 @@ #define MIPI_CSIS_ISPCFG_PIXEL_MODE_SINGLE (0 << 12) #define MIPI_CSIS_ISPCFG_PIXEL_MODE_DUAL (1 << 12) #define MIPI_CSIS_ISPCFG_PIXEL_MODE_QUAD (2 << 12) /* i.MX8M[MNP] only */ +#define MIPI_CSIS_ISPCFG_PIXEL_MASK (3 << 12) #define MIPI_CSIS_ISPCFG_ALIGN_32BIT BIT(11) #define MIPI_CSIS_ISPCFG_FMT(fmt) ((fmt) << 2) #define MIPI_CSIS_ISPCFG_FMT_MASK (0x3f << 2) @@ -506,7 +507,25 @@ static void __mipi_csis_set_format(struct csi_state *state) /* Color format */ val = mipi_csis_read(state, MIPI_CSIS_ISP_CONFIG_CH(0)); - val &= ~(MIPI_CSIS_ISPCFG_ALIGN_32BIT | MIPI_CSIS_ISPCFG_FMT_MASK); + val &= ~(MIPI_CSIS_ISPCFG_ALIGN_32BIT | MIPI_CSIS_ISPCFG_FMT_MASK + | MIPI_CSIS_ISPCFG_PIXEL_MASK); + + /* + * YUV 4:2:2 can be transferred with 8 or 16 bits per clock sample + * (referred to in the documentation as single and dual pixel modes + * respectively, although the 8-bit mode transfers half a pixel per + * clock sample and the 16-bit mode one pixel). While both mode work + * when the CSIS is connected to a receiver that supports either option, + * single pixel mode requires clock rates twice as high. As all SoCs + * that integrate the CSIS can operate in 16-bit bit mode, and some do + * not support 8-bit mode (this is the case of the i.MX8MP), use dual + * pixel mode unconditionally. + * + * TODO: Verify which other formats require DUAL (or QUAD) modes. + */ + if (state->csis_fmt->data_type == MIPI_CSI2_DATA_TYPE_YUV422_8) + val |= MIPI_CSIS_ISPCFG_PIXEL_MODE_DUAL; + val |= MIPI_CSIS_ISPCFG_FMT(state->csis_fmt->data_type); mipi_csis_write(state, MIPI_CSIS_ISP_CONFIG_CH(0), val); From 6d8d3c0953eb508a0f65859bb562c256dfb2d055 Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Mon, 21 Feb 2022 18:47:25 +0100 Subject: [PATCH 05/14] media: imx: imx-mipi-csis: Add RGB565_1X16 Add RGB565_1X16 to the enumeration of supported image formats. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Acked-by: Rui Miguel Silva Acked-by: Sakari Ailus Signed-off-by: Hans Verkuil --- drivers/media/platform/imx/imx-mipi-csis.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/media/platform/imx/imx-mipi-csis.c b/drivers/media/platform/imx/imx-mipi-csis.c index 5873ff74f737..a1bfc7689967 100644 --- a/drivers/media/platform/imx/imx-mipi-csis.c +++ b/drivers/media/platform/imx/imx-mipi-csis.c @@ -360,6 +360,12 @@ static const struct csis_pix_format mipi_csis_formats[] = { .data_type = MIPI_CSI2_DATA_TYPE_YUV422_8, .width = 16, }, + /* RGB formats. */ + { + .code = MEDIA_BUS_FMT_RGB565_1X16, + .data_type = MIPI_CSI2_DATA_TYPE_RGB565, + .width = 16, + }, /* RAW (Bayer and greyscale) formats. */ { .code = MEDIA_BUS_FMT_SBGGR8_1X8, From 28ba5e8024b9508df279c2172738d0c225c43496 Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Mon, 21 Feb 2022 18:47:26 +0100 Subject: [PATCH 06/14] media: imx: imx-mipi-csis: Add BGR888 Add support for the BGR888_1X24 image format. No existing media bus codes describe exactly the way data is transferred on the CSI-2 bus. This is not a new issue, the CSI-2 YUV422 8-bit format is described by MEDIA_BUS_FMT_UYVY8_1X16 which is an arbitrary convention and not an exact match. Use the MEDIA_BUS_FMT_BGR888_1X24 to follow the same convention, based on the order in which bits are transmitted over the CSI-2 bus. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Acked-by: Rui Miguel Silva Acked-by: Sakari Ailus Signed-off-by: Hans Verkuil --- drivers/media/platform/imx/imx-mipi-csis.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/media/platform/imx/imx-mipi-csis.c b/drivers/media/platform/imx/imx-mipi-csis.c index a1bfc7689967..027b622ccb97 100644 --- a/drivers/media/platform/imx/imx-mipi-csis.c +++ b/drivers/media/platform/imx/imx-mipi-csis.c @@ -365,6 +365,10 @@ static const struct csis_pix_format mipi_csis_formats[] = { .code = MEDIA_BUS_FMT_RGB565_1X16, .data_type = MIPI_CSI2_DATA_TYPE_RGB565, .width = 16, + }, { + .code = MEDIA_BUS_FMT_BGR888_1X24, + .data_type = MIPI_CSI2_DATA_TYPE_RGB888, + .width = 24, }, /* RAW (Bayer and greyscale) formats. */ { From 80ebe988cf8f322ee8803f33d0e5652d4e3142dc Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Mon, 21 Feb 2022 18:47:27 +0100 Subject: [PATCH 07/14] media: imx: imx-mipi-csis: Add output format Due to how pixel components are transmitted on the CSI-2 serial bus and how they are deserialized by the CSI-2 receiver, the component ordering might change and the image formats on the sink and source pads of the receiver should reflect it. For RGB24, in example, the component ordering on the wire as described by the CSI-2 specification matches the BGR888 format, while once deserialized by the CSIS receiver it matches the RGB888 format. Add an additional .output field to struct csis_pix_format to allow propagating the correct format to the source pad after a format configuration on the sink. The change is only relevant for RGB24 but paves the way for further format translations in future. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Acked-by: Rui Miguel Silva Acked-by: Sakari Ailus Signed-off-by: Hans Verkuil --- drivers/media/platform/imx/imx-mipi-csis.c | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/media/platform/imx/imx-mipi-csis.c b/drivers/media/platform/imx/imx-mipi-csis.c index 027b622ccb97..e3e0f4a70b3b 100644 --- a/drivers/media/platform/imx/imx-mipi-csis.c +++ b/drivers/media/platform/imx/imx-mipi-csis.c @@ -349,6 +349,7 @@ struct csi_state { struct csis_pix_format { u32 code; + u32 output; u32 data_type; u8 width; }; @@ -357,94 +358,116 @@ static const struct csis_pix_format mipi_csis_formats[] = { /* YUV formats. */ { .code = MEDIA_BUS_FMT_UYVY8_1X16, + .output = MEDIA_BUS_FMT_UYVY8_1X16, .data_type = MIPI_CSI2_DATA_TYPE_YUV422_8, .width = 16, }, /* RGB formats. */ { .code = MEDIA_BUS_FMT_RGB565_1X16, + .output = MEDIA_BUS_FMT_RGB565_1X16, .data_type = MIPI_CSI2_DATA_TYPE_RGB565, .width = 16, }, { .code = MEDIA_BUS_FMT_BGR888_1X24, + .output = MEDIA_BUS_FMT_RGB888_1X24, .data_type = MIPI_CSI2_DATA_TYPE_RGB888, .width = 24, }, /* RAW (Bayer and greyscale) formats. */ { .code = MEDIA_BUS_FMT_SBGGR8_1X8, + .output = MEDIA_BUS_FMT_SBGGR8_1X8, .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_SGBRG8_1X8, + .output = MEDIA_BUS_FMT_SGBRG8_1X8, .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_SGRBG8_1X8, + .output = MEDIA_BUS_FMT_SGRBG8_1X8, .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_SRGGB8_1X8, + .output = MEDIA_BUS_FMT_SRGGB8_1X8, .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_Y8_1X8, + .output = MEDIA_BUS_FMT_Y8_1X8, .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_SBGGR10_1X10, + .output = MEDIA_BUS_FMT_SBGGR10_1X10, .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_SGBRG10_1X10, + .output = MEDIA_BUS_FMT_SGBRG10_1X10, .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_SGRBG10_1X10, + .output = MEDIA_BUS_FMT_SGRBG10_1X10, .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_SRGGB10_1X10, + .output = MEDIA_BUS_FMT_SRGGB10_1X10, .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_Y10_1X10, + .output = MEDIA_BUS_FMT_Y10_1X10, .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_SBGGR12_1X12, + .output = MEDIA_BUS_FMT_SBGGR12_1X12, .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_SGBRG12_1X12, + .output = MEDIA_BUS_FMT_SGBRG12_1X12, .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_SGRBG12_1X12, + .output = MEDIA_BUS_FMT_SGRBG12_1X12, .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_SRGGB12_1X12, + .output = MEDIA_BUS_FMT_SRGGB12_1X12, .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_Y12_1X12, + .output = MEDIA_BUS_FMT_Y12_1X12, .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_SBGGR14_1X14, + .output = MEDIA_BUS_FMT_SBGGR14_1X14, .data_type = MIPI_CSI2_DATA_TYPE_RAW14, .width = 14, }, { .code = MEDIA_BUS_FMT_SGBRG14_1X14, + .output = MEDIA_BUS_FMT_SGBRG14_1X14, .data_type = MIPI_CSI2_DATA_TYPE_RAW14, .width = 14, }, { .code = MEDIA_BUS_FMT_SGRBG14_1X14, + .output = MEDIA_BUS_FMT_SGRBG14_1X14, .data_type = MIPI_CSI2_DATA_TYPE_RAW14, .width = 14, }, { .code = MEDIA_BUS_FMT_SRGGB14_1X14, + .output = MEDIA_BUS_FMT_SRGGB14_1X14, .data_type = MIPI_CSI2_DATA_TYPE_RAW14, .width = 14, } @@ -1107,6 +1130,9 @@ static int mipi_csis_set_fmt(struct v4l2_subdev *sd, CSIS_PAD_SOURCE); *fmt = sdformat->format; + /* The format on the source pad might change due to unpacking. */ + fmt->code = csis_fmt->output; + /* Store the CSIS format descriptor for active formats. */ if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE) state->csis_fmt = csis_fmt; From 0f6146d476fc99862899e70f2554ee77b444b7b9 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Fri, 18 Feb 2022 13:42:07 -0500 Subject: [PATCH 08/14] media: v4l2-core: Initialize h264 scaling matrix In the final H264 API, it is not required to set scaling matrix if they are not present in the bitstream. A flag was added in order to let the driver know. The downside is that it leaves the default control value to 0, which isn't valid. As per the spec (see formulas 7-8/7-9), when the scaling matrix are absent from the bitstream, flat values of 16 should be used. This improves this control semantic in a way that the control value are always valid. Drivers can then use the scaling_matrix control values without having to check its presence. Same method was employed for MPEG2_QUANTISATION. This fixes issues with MTK VCODEC H264 decoder when using GStreamer. GStreamer does not set this control if its not present in the bitstream. As MTK VDCODEC was using the initialized to 0 values, the frames ended up completely gray. Fixes: 54889c51b833d236 ("media: uapi: h264: Rename and clarify PPS_FLAG_SCALING_MATRIX_PRESENT") Signed-off-by: Nicolas Dufresne Tested-by: AngeloGioacchino Del Regno Signed-off-by: Hans Verkuil --- drivers/media/v4l2-core/v4l2-ctrls-core.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c index 85c2d3f39d96..18194a1dd893 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls-core.c +++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c @@ -113,6 +113,7 @@ static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx, struct v4l2_ctrl_mpeg2_quantisation *p_mpeg2_quant; struct v4l2_ctrl_vp8_frame *p_vp8_frame; struct v4l2_ctrl_fwht_params *p_fwht_params; + struct v4l2_ctrl_h264_scaling_matrix *p_h264_scaling_matrix; void *p = ptr.p + idx * ctrl->elem_size; if (ctrl->p_def.p_const) @@ -160,6 +161,15 @@ static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx, p_fwht_params->flags = V4L2_FWHT_FL_PIXENC_YUV | (2 << V4L2_FWHT_FL_COMPONENTS_NUM_OFFSET); break; + case V4L2_CTRL_TYPE_H264_SCALING_MATRIX: + p_h264_scaling_matrix = p; + /* + * The default (flat) H.264 scaling matrix when none are + * specified in the bitstream, this is according to formulas + * (7-8) and (7-9) of the specification. + */ + memset(p_h264_scaling_matrix, 16, sizeof(*p_h264_scaling_matrix)); + break; } } From 884c8bd91828d851ede5489cb5b4134ffc61f965 Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Mon, 21 Feb 2022 19:17:08 +0100 Subject: [PATCH 09/14] media: imx: imx8mq-mipi-csi2: Remove YUV422 2X8 The 2X8 variants of MEDIA_BUS_FMT_YUYV8_2X8 does not apply to serial busses. Drop it and while at it also add the canonical UYVY wire format for packed YUV422 when transmitted on the CSI-2 serial bus. Also beautify a little the formats declaration list by putting the opening curly brace after the comment. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Signed-off-by: Hans Verkuil --- drivers/staging/media/imx/imx8mq-mipi-csi2.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/staging/media/imx/imx8mq-mipi-csi2.c b/drivers/staging/media/imx/imx8mq-mipi-csi2.c index 56ef3b3b2906..449796a0b942 100644 --- a/drivers/staging/media/imx/imx8mq-mipi-csi2.c +++ b/drivers/staging/media/imx/imx8mq-mipi-csi2.c @@ -200,12 +200,13 @@ static const struct csi2_pix_format imx8mq_mipi_csi_formats[] = { }, { .code = MEDIA_BUS_FMT_SRGGB14_1X14, .width = 14, - }, { + }, /* YUV formats */ - .code = MEDIA_BUS_FMT_YUYV8_2X8, + { + .code = MEDIA_BUS_FMT_YUYV8_1X16, .width = 16, }, { - .code = MEDIA_BUS_FMT_YUYV8_1X16, + .code = MEDIA_BUS_FMT_UYVY8_1X16, .width = 16, } }; From 9c634d6d137fa2024579b2c23fa3f19053e7a910 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 21 Feb 2022 14:12:01 -0500 Subject: [PATCH 10/14] doc: media: Document MM21 tiled format MM21 has been documented through two software implementation [0] in libyuv and [2] GStreamer. This format is similar to other tiled format, though it uses uneven tile sizes. The luma tiles are 16x32 while the chroma tile have a subsampled size of 16x16. This is the uncompressed cousin of V4L2_PIX_FMT_MT21C and shares its tiling pattern and alignment. [0] https://chromium.googlesource.com/libyuv/libyuv/+/b4ddbaf549a1bf5572bf703fd2862d1eb7380c6a [1] https://gitlab.freedesktop.org/gstreamer/gstreamer/-/commit/c9b127dae3af4b3fe78468e697cb491902425254 Signed-off-by: Nicolas Dufresne Reviewed-by: Ezequiel Garcia Signed-off-by: Hans Verkuil --- .../userspace-api/media/v4l/pixfmt-reserved.rst | 15 ++++----------- .../userspace-api/media/v4l/pixfmt-yuv-planar.rst | 6 ++++++ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Documentation/userspace-api/media/v4l/pixfmt-reserved.rst b/Documentation/userspace-api/media/v4l/pixfmt-reserved.rst index 2f2133b4cd9c..cabfa34b7db5 100644 --- a/Documentation/userspace-api/media/v4l/pixfmt-reserved.rst +++ b/Documentation/userspace-api/media/v4l/pixfmt-reserved.rst @@ -233,19 +233,12 @@ please make a proposal on the linux-media mailing list. - ``V4L2_PIX_FMT_MT21C`` - 'MT21' - - Compressed two-planar YVU420 format used by Mediatek MT8173. - The compression is lossless. - It is an opaque intermediate format and the MDP hardware must be + - Compressed two-planar YVU420 format used by Mediatek MT8173, MT8192, + MT8195 and more. The compression is lossless. This format have + similitude with ``V4L2_PIX_FMT_MM21`` in term of alignment and tiling. + It remains an opaque intermediate format and the MDP hardware must be used to convert ``V4L2_PIX_FMT_MT21C`` to ``V4L2_PIX_FMT_NV12M``, ``V4L2_PIX_FMT_YUV420M`` or ``V4L2_PIX_FMT_YVU420``. - * .. _V4L2-PIX-FMT-MM21: - - - ``V4L2_PIX_FMT_MM21`` - - 'MM21' - - Non-compressed, tiled two-planar format used by Mediatek MT8183. - This is an opaque intermediate format and the MDP3 hardware can be - used to convert it to other formats. - .. raw:: latex \normalsize diff --git a/Documentation/userspace-api/media/v4l/pixfmt-yuv-planar.rst b/Documentation/userspace-api/media/v4l/pixfmt-yuv-planar.rst index 3a09d93d405b..570791fbc5c8 100644 --- a/Documentation/userspace-api/media/v4l/pixfmt-yuv-planar.rst +++ b/Documentation/userspace-api/media/v4l/pixfmt-yuv-planar.rst @@ -296,6 +296,12 @@ tiles linearly in memory. The line stride and image height must be aligned to a multiple of 32. The layouts of the luma and chroma planes are identical. +``V4L2_PIX_FMT_MM21`` store luma pixel in 16x32 tiles, and chroma pixels +in 16x16 tiles. The line stride must be aligned to a multiple of 16 and the +image height must be aligned to a multiple of 32. The number of luma and chroma +tiles are identical, even though the tile size differ. The image is formed of +two non-contiguous planes. + .. _nv12mt: .. kernel-figure:: nv12mt.svg From 88c63ac741266bf3fc0812578199dfb16a3c31b5 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 21 Feb 2022 14:37:27 -0500 Subject: [PATCH 11/14] doc: media: Document VP9 reference_mode miss-placement The reference_mode derived syntax is part of the compressed headers and should have been moved into the corresponding control structure. Document this mistake. The value can be set to 0 if the driver does not require compressed headers information. Signed-off-by: Nicolas Dufresne Reviewed-by: Andrzej Pietrasiewicz Signed-off-by: Hans Verkuil [hverkuil: fix small typo in comment] --- .../userspace-api/media/v4l/ext-ctrls-codec-stateless.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst index cc080c4257d0..c6ef05e0d3a4 100644 --- a/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst +++ b/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst @@ -1692,7 +1692,12 @@ See section '7.3.1 Tx mode semantics' of the :ref:`vp9` specification for more d * - __u8 - ``reference_mode`` - Specifies the type of inter prediction to be used. See - :ref:`Reference Mode` for more details. + :ref:`Reference Mode` for more details. Note that + this is derived as part of the compressed header parsing process and + for this reason should have been part of + :c:type: `v4l2_ctrl_vp9_compressed_hdr` optional control. It is safe to + set this value to zero if the driver does not require compressed + headers. * - __u8 - ``reserved[7]`` - Applications and drivers must set this to zero. From 5c0701a0e791868f7dbf20111c18e08ab6a867a9 Mon Sep 17 00:00:00 2001 From: Jacopo Mondi Date: Tue, 22 Feb 2022 22:46:43 +0100 Subject: [PATCH 12/14] media: imx: csis: Store pads format separately As the formats on the sink and source pad might be different store them separately. The pad format is used to configure the image width and height in mipi_csis_system_enable(). As the csis cannot downscale, using the sink or the source one isn't relevant. Signed-off-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Signed-off-by: Hans Verkuil --- drivers/media/platform/imx/imx-mipi-csis.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/imx/imx-mipi-csis.c b/drivers/media/platform/imx/imx-mipi-csis.c index e3e0f4a70b3b..0a72734db55e 100644 --- a/drivers/media/platform/imx/imx-mipi-csis.c +++ b/drivers/media/platform/imx/imx-mipi-csis.c @@ -330,7 +330,7 @@ struct csi_state { struct mutex lock; /* Protect csis_fmt, format_mbus and state */ const struct csis_pix_format *csis_fmt; - struct v4l2_mbus_framefmt format_mbus; + struct v4l2_mbus_framefmt format_mbus[CSIS_PADS_NUM]; u32 state; spinlock_t slock; /* Protect events */ @@ -535,7 +535,7 @@ static void mipi_csis_system_enable(struct csi_state *state, int on) /* Called with the state.lock mutex held */ static void __mipi_csis_set_format(struct csi_state *state) { - struct v4l2_mbus_framefmt *mf = &state->format_mbus; + struct v4l2_mbus_framefmt *mf = &state->format_mbus[CSIS_PAD_SINK]; u32 val; /* Color format */ @@ -967,7 +967,7 @@ mipi_csis_get_format(struct csi_state *state, if (which == V4L2_SUBDEV_FORMAT_TRY) return v4l2_subdev_get_try_format(&state->sd, sd_state, pad); - return &state->format_mbus; + return &state->format_mbus[pad]; } static int mipi_csis_init_cfg(struct v4l2_subdev *sd, From 17e83a0a1e134401616120b6976f65234e7ccc45 Mon Sep 17 00:00:00 2001 From: Jernej Skrabec Date: Tue, 22 Feb 2022 19:03:28 +0100 Subject: [PATCH 13/14] media: hantro: sunxi: Fix VP9 steps It turns out that postproc on Allwinner H6 needs width and height to be multiple of 32. Fixes: 86790a4fdf4b ("media: hantro: Add support for Allwinner H6") Signed-off-by: Jernej Skrabec Signed-off-by: Hans Verkuil --- drivers/staging/media/hantro/sunxi_vpu_hw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/staging/media/hantro/sunxi_vpu_hw.c b/drivers/staging/media/hantro/sunxi_vpu_hw.c index 90633406c4eb..c0edd5856a0c 100644 --- a/drivers/staging/media/hantro/sunxi_vpu_hw.c +++ b/drivers/staging/media/hantro/sunxi_vpu_hw.c @@ -29,10 +29,10 @@ static const struct hantro_fmt sunxi_vpu_dec_fmts[] = { .frmsize = { .min_width = 48, .max_width = 3840, - .step_width = MB_DIM, + .step_width = 32, .min_height = 48, .max_height = 2160, - .step_height = MB_DIM, + .step_height = 32, }, }, }; From 0bff66dcbe55cd4b13dfe53abad9268e34d48cbf Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 22 Feb 2022 23:23:31 +0100 Subject: [PATCH 14/14] media: cec: seco: Drop pointless include This driver uses GPIO descriptors not the old legacy GPIO API so stop including . Fix a bug using a completely unrelated legacy API flag GPIOF_IN by switching to the actually desired flag GPIOD_IN. Signed-off-by: Linus Walleij Signed-off-by: Hans Verkuil --- drivers/media/cec/platform/seco/seco-cec.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/media/cec/platform/seco/seco-cec.c b/drivers/media/cec/platform/seco/seco-cec.c index ae138cc253fd..4df56096a476 100644 --- a/drivers/media/cec/platform/seco/seco-cec.c +++ b/drivers/media/cec/platform/seco/seco-cec.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -551,7 +550,7 @@ static int secocec_acpi_probe(struct secocec_data *sdev) struct gpio_desc *gpio; int irq = 0; - gpio = devm_gpiod_get(dev, NULL, GPIOF_IN); + gpio = devm_gpiod_get(dev, NULL, GPIOD_IN); if (IS_ERR(gpio)) { dev_err(dev, "Cannot request interrupt gpio"); return PTR_ERR(gpio);